use tokio::time::Instant;
use super::*;
use crate::scheduler::IDLE_TIMEOUT;
use crate::scheduler::engine::STOP_DEADLINE;
async fn idle_runner_beside_a_queued_job(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
idle_runner(github, docker);
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.capture_end = CaptureEnd::Container;
}
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(5)];
state.queued_on.insert(5, "o/s".to_owned());
}
let (mut engine, handle) = engine_with(github, docker, dir, &two_repositories());
engine.recover().await.unwrap();
for occupant in engine.occupants_mut() {
occupant.started_at -= IDLE_TIMEOUT;
}
(engine, handle)
}
#[tokio::test(start_paused = true)]
async fn removing_an_idle_runner_does_not_hold_back_a_start_for_another_repository() {
let github = FakeGithub::default();
let docker = FakeDocker::default();
let dir = TestDir::new("pacing");
let (mut engine, handle) = idle_runner_beside_a_queued_job(&github, &docker, &dir).await;
let began = Instant::now();
engine.tick(Shutdown::No).await;
let took = began.elapsed();
assert_eq!(removed(&docker.calls()), ["remove container-1"]);
assert_eq!(registered(&github.calls()).len(), 1, "o/s got its runner");
assert!(
took < super::super::history::CAPTURE_DRAIN,
"the tick took {took:?}"
);
handle.store.flush().await;
let history = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
let runners: Vec<(String, Option<String>, Option<String>)> = history
.prepare("SELECT repository, purpose, stopped_for FROM runners ORDER BY started_at")
.unwrap()
.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
.unwrap()
.collect::<Result<_, _>>()
.unwrap();
assert_eq!(
runners,
[
("o/r".to_owned(), None, Some("expired".to_owned())),
("o/s".to_owned(), Some("job".to_owned()), None)
]
);
}
#[derive(Debug, Default)]
struct Seen {
waits: Vec<Duration>,
turnovers: Vec<(Instant, Duration)>,
starts: usize,
taken: usize,
over_budget: Vec<u64>,
}
struct Running {
container: String,
name: String,
job: u64,
named_at: Instant,
ends_at: Instant,
}
struct Burst {
github: FakeGithub,
docker: FakeDocker,
boot: Duration,
run: Duration,
named_after: Duration,
budget: u64,
slow_removals: bool,
queued_at: HashMap<u64, Instant>,
listed: BTreeSet<u64>,
listings: usize,
listed_at: Vec<Instant>,
calls_seen: usize,
listed_at_of: BTreeMap<String, Vec<Instant>>,
running: Vec<Running>,
started: BTreeSet<String>,
assigned: BTreeSet<String>,
freed: VecDeque<Instant>,
seen: Seen,
}
impl Burst {
fn new(github: &FakeGithub, docker: &FakeDocker, budget: u64) -> Burst {
Burst {
github: github.clone(),
docker: docker.clone(),
boot: Duration::from_secs(6),
run: Duration::from_secs(10),
named_after: Duration::from_secs(1),
budget,
slow_removals: false,
queued_at: HashMap::new(),
listed: BTreeSet::new(),
listings: 0,
listed_at: Vec::new(),
calls_seen: 0,
listed_at_of: BTreeMap::new(),
running: Vec::new(),
started: BTreeSet::new(),
assigned: BTreeSet::new(),
freed: VecDeque::new(),
seen: Seen::default(),
}
}
fn queue(&mut self, count: u64) {
let mut github = self.github.0.lock().unwrap();
let first = github
.queued
.iter()
.map(|job| job.id)
.max()
.unwrap_or(0)
.max(self.queued_at.keys().copied().max().unwrap_or(0))
+ 1;
for id in first..first + count {
github.queued.push(job(id));
github.queued_on.insert(id, "o/r".to_owned());
self.queued_at.insert(id, Instant::now());
}
}
fn listed_since(&self, repository: &str, from: Instant) -> Vec<Instant> {
self.listed_at_of
.get(repository)
.into_iter()
.flatten()
.copied()
.filter(|at| *at >= from)
.collect()
}
fn done(&self) -> bool {
self.seen.taken == self.queued_at.len() && self.running.is_empty()
}
fn step(&mut self, now: Instant) {
let (github, docker) = (self.github.clone(), self.docker.clone());
let mut github = github.0.lock().unwrap();
let mut docker = docker.0.lock().unwrap();
let listings = github
.calls
.iter()
.filter(|call| call.starts_with("queued "))
.count();
if listings > self.listings {
self.listings = listings;
self.listed_at.push(now);
self.listed.extend(github.queued.iter().map(|job| job.id));
}
for call in &github.calls[self.calls_seen..] {
if let Some(repository) = call.strip_prefix("queued ") {
let times = self.listed_at_of.entry(repository.to_owned()).or_default();
if times.last() != Some(&now) {
times.push(now);
}
}
}
self.calls_seen = github.calls.len();
self.see_starts(&mut docker, now);
self.take_jobs(&mut github, &mut docker, now);
self.end_jobs(&mut github, &mut docker, now);
}
fn see_starts(&mut self, docker: &mut DockerState, now: Instant) {
let new: Vec<String> = docker
.containers
.iter()
.filter(|container| {
container.labels.contains_key("runnerd.job")
&& !self.started.contains(&container.id)
})
.map(|container| container.id.clone())
.collect();
for id in new {
self.started.insert(id.clone());
self.seen.starts += 1;
if self.slow_removals {
let step = u64::try_from(self.seen.starts * 7 % 5).unwrap();
docker
.slow_removes
.insert(id.clone(), Duration::from_secs(5 + step * 5));
}
let using: u64 = docker
.containers
.iter()
.filter(|container| {
container.running && container.labels.contains_key("runnerd.job")
})
.filter_map(|container| docker.memory.get(&container.id))
.sum();
if using > self.budget {
self.seen.over_budget.push(using);
}
let name = docker
.containers
.iter()
.find(|container| container.id == id)
.map(|container| container.name.clone())
.unwrap_or_default();
let at = docker.started_on_clock.get(&name).copied().unwrap_or(now);
if let Some(stopped) = self.freed.pop_front() {
self.seen
.turnovers
.push((stopped, at.saturating_duration_since(stopped)));
}
}
}
fn take_jobs(&mut self, github: &mut GithubState, docker: &mut DockerState, now: Instant) {
let idle: Vec<(String, String)> = docker
.containers
.iter()
.filter(|container| {
container.running
&& container.labels.contains_key("runnerd.job")
&& !self.assigned.contains(&container.id)
&& docker
.started_on_clock
.get(&container.name)
.is_some_and(|at| now.saturating_duration_since(*at) >= self.boot)
})
.map(|container| (container.id.clone(), container.name.clone()))
.collect();
for (container, name) in idle {
let Some(job) = github.queued.iter().map(|job| job.id).min() else {
break;
};
github.queued.retain(|queued| queued.id != job);
for runner in &mut github.runners {
if runner.name == name {
runner.busy = true;
}
}
let mut details = details(job, "in_progress", None);
details.runner_name = Some(name.clone());
github.jobs.insert(job, details);
let started = docker.started_on_clock.get(&name).copied().unwrap_or(now);
self.seen
.waits
.push(started.saturating_duration_since(self.queued_at[&job]));
self.seen.taken += 1;
self.assigned.insert(container.clone());
self.running.push(Running {
container,
name,
job,
named_at: now + self.named_after,
ends_at: now + self.run + self.run * u32::try_from(job * 7 % 11).unwrap() / 10,
});
}
}
fn end_jobs(&mut self, github: &mut GithubState, docker: &mut DockerState, now: Instant) {
for running in &self.running {
if running.named_at <= now {
docker
.container_captures
.entry(running.container.clone())
.or_insert_with(|| format!("@@job {0} {0}\n", running.job));
}
}
let (ended, left): (Vec<Running>, Vec<Running>) = std::mem::take(&mut self.running)
.into_iter()
.partition(|running| running.ends_at <= now);
self.running = left;
let idle = docker
.containers
.iter()
.filter(|container| {
container.running
&& container.labels.contains_key("runnerd.job")
&& !self.assigned.contains(&container.id)
})
.count();
let listed = github
.queued
.iter()
.filter(|job| self.listed.contains(&job.id))
.count();
let waiting = listed > idle + 1;
for running in ended {
github.runners.retain(|runner| runner.name != running.name);
let mut details = details(running.job, "completed", Some("success"));
details.runner_name = Some(running.name.clone());
github.jobs.insert(running.job, details);
if let Some(container) = docker
.containers
.iter_mut()
.find(|container| container.id == running.container)
{
container.running = false;
}
for id in [
running.container.as_str(),
running.container.as_str(),
"not-a-runner",
] {
docker
.events
.retain(|events| events.send(Ok(id.to_owned())).is_ok());
}
if waiting {
self.freed.push_back(now);
}
}
}
}
fn burst_fakes(listing_lag: Duration) -> (FakeGithub, FakeDocker) {
let github = FakeGithub::default();
let docker = FakeDocker::default();
{
let mut state = github.0.lock().unwrap();
state.strict = true;
state.latency = Duration::from_millis(300);
state.listing_latency = listing_lag;
}
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.capture_end = CaptureEnd::Container;
state.measured = Some(format!("slot 1 changed {} bytes\n", 1 << 20));
state.latency = DockerLatency {
start: Duration::from_millis(1000),
contention: Duration::ZERO,
remove_running: Duration::from_millis(1000),
remove_stopped: Duration::from_millis(300),
prepare: Duration::ZERO,
};
}
(github, docker)
}
async fn play(burst: &mut Burst, mut until: impl FnMut(&Burst, Instant) -> bool) {
let began = Instant::now();
loop {
let now = Instant::now();
burst.step(now);
if until(burst, now) || now - began > Duration::from_mins(30) {
return;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
#[tokio::test(start_paused = true)]
async fn a_slot_whose_job_ended_takes_the_next_listed_job_within_seconds() {
const JOBS: u64 = 16;
const G: u64 = 1 << 30;
let (github, docker) = burst_fakes(Duration::from_secs(3));
let dir = TestDir::new("turnover");
let config = CONFIG.replace("slots = 2", "slots = 4");
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(12 * G + G / 2);
let mut burst = Burst::new(&github, &docker, 12 * G + G / 2);
burst.queue(JOBS);
let run = tokio::spawn(engine.run());
let broke = Instant::now() + Duration::from_secs(40);
play(&mut burst, |burst, now| {
now >= broke && !burst.running.is_empty()
})
.await;
{
docker.0.lock().unwrap().fail_events = 2;
docker.break_events();
}
play(&mut burst, Burst::done_at).await;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
let seen = &burst.seen;
assert_eq!(
seen.taken,
usize::try_from(JOBS).unwrap(),
"every job ran: {seen:?}"
);
assert_eq!(
seen.starts,
usize::try_from(JOBS).unwrap(),
"no job got a second runner: {seen:?}"
);
assert!(
seen.over_budget.is_empty(),
"starts passed the slice: {seen:?}"
);
let followed: Vec<Duration> = seen
.turnovers
.iter()
.filter(|(stopped, _)| *stopped < broke || *stopped > broke + Duration::from_secs(10))
.map(|(_, took)| *took)
.collect();
assert!(followed.len() >= 8, "too few slots turned over: {seen:?}");
assert!(
followed.iter().all(|took| *took <= Duration::from_secs(4)),
"a slot waited for its next runner: {followed:?}"
);
assert!(
seen.turnovers
.iter()
.all(|(_, took)| *took <= Duration::from_secs(20)),
"a slot freed while Docker's events were down waited past a poll: {:?}",
seen.turnovers
);
assert!(
docker
.0
.lock()
.unwrap()
.events
.iter()
.all(tokio::sync::mpsc::UnboundedSender::is_closed),
"the engine stopped following Docker's events as it stopped"
);
}
async fn stored(store: &Store) {
let store = store.clone();
let runtime = tokio::runtime::Handle::current();
tokio::task::spawn_blocking(move || runtime.block_on(store.flush()))
.await
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_job_listed_beside_running_jobs_starts_without_waiting_for_their_reads() {
fn took(burst: &Burst, queued: Instant, nth: usize) -> Duration {
let started = queued + burst.seen.waits[nth];
let listed = burst
.listed_at
.iter()
.copied()
.find(|at| *at >= queued)
.unwrap();
started.saturating_duration_since(listed)
}
let (github, docker) = burst_fakes(Duration::ZERO);
github.0.lock().unwrap().latency = Duration::from_millis(1800);
let dir = TestDir::new("listed-start");
let config = CONFIG.replace("slots = 2", "slots = 8");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_hours(1);
burst.queue(6);
let run = tokio::spawn(engine.run());
let settled = Instant::now() + Duration::from_mins(2);
play(&mut burst, |_, now| now >= settled).await;
assert_eq!(burst.running.len(), 6, "six jobs run: {:?}", burst.seen);
burst.queue(1);
let seventh = Instant::now();
play(&mut burst, |burst, _| burst.seen.taken == 7).await;
let named = Instant::now() + burst.named_after;
play(&mut burst, |_, now| now >= named).await;
stored(&handle.store).await;
burst.queue(1);
let eighth = Instant::now();
play(&mut burst, |burst, _| burst.seen.taken == 8).await;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
for (nth, queued) in [(6, seventh), (7, eighth)] {
let took = took(&burst, queued, nth);
assert!(
took <= Duration::from_secs(3),
"job {}'s runner started {took:?} after the listing that showed it",
nth + 1
);
}
}
#[tokio::test(start_paused = true)]
async fn slow_job_reads_do_not_push_the_next_listing_out() {
async fn gaps(burst: &mut Burst, repository: &str, span: Duration) -> Vec<Duration> {
let from = Instant::now();
play(burst, |_, now| now >= from + span).await;
gaps_of(&burst.listed_since(repository, from))
}
let (github, docker) = burst_fakes(Duration::ZERO);
github.0.lock().unwrap().job_latency = Duration::from_secs(9);
let dir = TestDir::new("fixed-rate");
let config = two_repositories().replace("slots = 2", "slots = 4");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_hours(1);
burst.queue(4);
let run = tokio::spawn(engine.run());
let settled = Instant::now() + Duration::from_mins(2);
play(&mut burst, |_, now| now >= settled).await;
assert_eq!(burst.running.len(), 4, "four jobs run: {:?}", burst.seen);
let read = github.0.lock().unwrap().job_reads;
let from = Instant::now();
let paced = gaps(&mut burst, "o/s", Duration::from_mins(3)).await;
let active = gaps_of(&burst.listed_since("o/r", from));
assert!(
github.0.lock().unwrap().job_reads > read,
"the ticks read the running jobs"
);
assert!(paced.len() >= 10, "too few listings: {paced:?}");
assert!(
paced
.iter()
.all(|gap| gap.abs_diff(Duration::from_secs(15)) <= Duration::from_millis(400)),
"o/s's listings left `poll_interval`'s pace: {paced:?}"
);
assert!(
active
.iter()
.all(|gap| *gap >= Duration::from_millis(4_900)),
"o/r's listings came closer than a third of `poll_interval`: {active:?}"
);
github.0.lock().unwrap().job_latency = Duration::from_secs(20);
let slow = gaps(&mut burst, "o/s", Duration::from_mins(3)).await;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
assert!(slow.len() >= 5, "too few listings: {slow:?}");
assert!(
slow.iter()
.skip(1)
.all(|gap| *gap >= Duration::from_millis(22_300)),
"ticks ran back to back: {slow:?}"
);
}
#[tokio::test(start_paused = true)]
async fn running_jobs_are_read_every_poll_interval_whatever_the_ticks() {
let (github, docker) = burst_fakes(Duration::ZERO);
let dir = TestDir::new("running-reads");
let config = CONFIG.replace("slots = 2", "slots = 4");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_hours(1);
burst.queue(4);
let run = tokio::spawn(engine.run());
let settled = Instant::now() + Duration::from_mins(2);
play(&mut burst, |_, now| now >= settled).await;
assert_eq!(burst.running.len(), 4, "four jobs run: {:?}", burst.seen);
let polled = polls(&github.calls());
let span = Duration::from_mins(3);
let until = Instant::now() + span;
let mut last = github.0.lock().unwrap().job_reads;
let (mut wake, mut reads) = (Instant::now(), vec![]);
while Instant::now() < until {
burst.step(Instant::now());
if Instant::now() >= wake {
handle.control.wake.notify_one();
wake += Duration::from_secs(5);
}
let now = github.0.lock().unwrap().job_reads;
if now > last {
if reads.last().is_none_or(|&(at, _): &(Instant, usize)| {
Instant::now() - at > Duration::from_secs(5)
}) {
reads.push((Instant::now(), 0));
}
if let Some((_, count)) = reads.last_mut() {
*count += now - last;
}
last = now;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
let ticks = polls(&github.calls()) - polled;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
assert!(ticks >= 30, "the ticks came every 5 s: {ticks} in {span:?}");
let gaps: Vec<Duration> = reads.windows(2).map(|pair| pair[1].0 - pair[0].0).collect();
assert!(
reads.len() >= 10 && reads.iter().all(|&(_, count)| count == 4),
"each round reads the four jobs once: {reads:?}"
);
assert!(
gaps.iter()
.all(|gap| *gap >= Duration::from_millis(14_500) && *gap <= Duration::from_secs(16)),
"rounds of reads left `poll_interval`'s pace: {gaps:?}"
);
}
#[tokio::test]
async fn a_job_no_listing_named_is_read_before_each_plan_inside_the_interval() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
running_job(&github, &docker, "o/r", 1, 1);
github.0.lock().unwrap().failed_jobs.insert(1);
let dir = TestDir::new("unnamed-read");
let (mut engine, _handle) = engine_in(&github, &docker, &dir);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
let first = github.0.lock().unwrap().job_reads;
assert!(first > 0, "the first tick reads the job, which fails");
github.0.lock().unwrap().failed_jobs.clear();
engine.tick(Shutdown::No).await;
let second = github.0.lock().unwrap().job_reads;
assert_eq!(second, first + 1, "the next tick reads it before its plan");
engine.tick(Shutdown::No).await;
assert_eq!(
github.0.lock().unwrap().job_reads,
second,
"named now, it is read again only at the poll's pace"
);
}
#[tokio::test(start_paused = true)]
async fn a_slow_collect_does_not_bring_the_next_listing_sooner() {
let github = FakeGithub::default();
let docker = FakeDocker::default();
running_job(&github, &docker, "o/r", 1, 1);
docker
.0
.lock()
.unwrap()
.slow_removes
.insert("container-1".to_owned(), Duration::from_secs(8));
let dir = TestDir::new("slow-collect");
let (engine, handle) = engine_with(&github, &docker, &dir, CONFIG);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
handle
.control
.pause(one("o/r"), RunningJobs::Kill)
.await
.unwrap();
tokio::time::sleep(Duration::from_secs(1)).await;
let mut listed_at = Vec::new();
let mut listings = github
.calls()
.iter()
.filter(|call| call.starts_with("queued "))
.count();
let until =
Instant::now() + crate::scheduler::engine::control::CANCEL_GRACE + Duration::from_mins(1);
while Instant::now() < until {
let now = github
.calls()
.iter()
.filter(|call| call.starts_with("queued "))
.count();
if now > listings {
listings = now;
listed_at.push(Instant::now());
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
handle.shutdown(Shutdown::Force);
run.await.unwrap();
assert_eq!(removed(&docker.calls()), ["remove container-1"]);
let gaps: Vec<Duration> = listed_at.windows(2).map(|pair| pair[1] - pair[0]).collect();
assert!(gaps.len() >= 8, "too few listings: {gaps:?}");
assert!(
gaps.iter().all(|gap| *gap >= Duration::from_millis(14_900)),
"two listings came closer than `poll_interval`: {gaps:?}"
);
}
#[tokio::test(start_paused = true)]
async fn overlapping_slow_releases_leave_the_polls_and_commands_their_turn() {
const JOBS: u64 = 48;
let (github, docker) = burst_fakes(Duration::ZERO);
let dir = TestDir::new("overlapping-releases");
let config = CONFIG.replace("slots = 2", "slots = 4");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.boot = Duration::from_secs(1);
burst.run = Duration::from_secs(3);
burst.slow_removals = true;
burst.queue(JOBS);
let run = tokio::spawn(engine.run());
let command = async {
tokio::time::sleep(Duration::from_mins(1)).await;
let sent = Instant::now();
handle
.control
.replace_config(
Config::parse(&config).unwrap(),
None,
None,
Commit::new(|| Ok(None)),
RunningSlots::Keep,
Author::Owner,
)
.await
.unwrap();
sent.elapsed()
};
let ((), answered) = tokio::join!(play(&mut burst, Burst::done_at), command);
let drained = Instant::now();
handle.shutdown(Shutdown::Force);
run.await.unwrap();
assert_eq!(
burst.seen.taken,
usize::try_from(JOBS).unwrap(),
"every job ran: {:?}",
burst.seen
);
assert!(
answered < Duration::from_secs(90),
"the command waited {answered:?}"
);
let gaps: Vec<Duration> = burst
.listed_at
.windows(2)
.map(|pair| pair[1] - pair[0])
.chain(burst.listed_at.last().map(|last| drained - *last))
.collect();
assert!(
gaps.iter().all(|gap| *gap < Duration::from_secs(90)),
"a poll waited for the backlog: {gaps:?}"
);
}
fn runner_containers(handle: &Handle) -> Vec<String> {
handle.status.borrow().profiles[0]
.slots
.iter()
.filter_map(|slot| slot.runner.as_ref().map(|runner| runner.container.clone()))
.collect()
}
#[tokio::test(start_paused = true)]
async fn runners_that_stop_at_once_or_are_removed_free_their_slots_before_the_next_poll() {
let github = FakeGithub::default();
let docker = FakeDocker::default();
github.0.lock().unwrap().queued = vec![job(1), job(2)];
docker.0.lock().unwrap().exit_at_start = 1;
let dir = TestDir::new("early-exits");
let config = CONFIG.replace("[host]\n", "[host]\npoll_interval = \"1h\"\n");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let run = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_secs(5)).await;
let left = runner_containers(&handle);
assert_eq!(
left.len(),
1,
"the runner that stopped as it started left its slot: {:?}",
docker.calls()
);
{
let mut state = docker.0.lock().unwrap();
state.containers.retain(|container| container.id != left[0]);
state
.events
.retain(|events| events.send(Ok(left[0].clone())).is_ok());
}
tokio::time::sleep(Duration::from_secs(2)).await;
let now = runner_containers(&handle);
assert!(
!now.contains(&left[0]),
"the removed runner left its slot: {now:?}"
);
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
impl Burst {
fn done_at(&self, _: Instant) -> bool {
self.done()
}
}
fn figures(mut durations: Vec<Duration>) -> String {
durations.sort();
let at = |percent: usize| {
durations[(durations.len() * percent / 100).min(durations.len() - 1)].as_secs_f64()
};
format!(
"p50 {:.1}s, p90 {:.1}s, max {:.1}s over {}",
at(50),
at(90),
at(100),
durations.len()
)
}
const CONTENTION_PLACEHOLDER: Duration = Duration::from_millis(250);
struct Staircase {
places: Vec<Duration>,
most_registering: usize,
most_starts: usize,
}
async fn burst_into_free_slots(
jobs: u64,
contention: Duration,
slow_first: Option<Duration>,
) -> Staircase {
let (github, docker) = burst_fakes(Duration::ZERO);
{
let mut state = docker.0.lock().unwrap();
state.latency.contention = contention;
state.slow_start = slow_first;
}
let dir = TestDir::new("staircase");
let config = CONFIG.replace("slots = 2", "slots = 16");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_hours(1);
burst.queue(jobs);
let queued = Instant::now();
let run = tokio::spawn(engine.run());
let all = usize::try_from(jobs).unwrap();
play(&mut burst, |burst, _| burst.seen.taken == all).await;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
let listed = burst.listed_at[0];
let mut places: Vec<Duration> = burst
.seen
.waits
.iter()
.map(|wait| (queued + *wait).saturating_duration_since(listed))
.collect();
places.sort();
let most_registering = github.0.lock().unwrap().most_registering;
let most_starts = docker.0.lock().unwrap().most_runner_starts;
Staircase {
places,
most_registering,
most_starts,
}
}
async fn staircase(jobs: u64) -> Vec<Duration> {
burst_into_free_slots(jobs, Duration::ZERO, None)
.await
.places
}
fn tenths(places: &[Duration]) -> String {
places
.iter()
.map(|place| format!("{:.1}", place.as_secs_f64()))
.collect::<Vec<_>>()
.join(", ")
}
#[tokio::test(start_paused = true)]
async fn a_burst_into_free_slots_starts_one_registration_at_a_time() {
let burst = burst_into_free_slots(12, Duration::ZERO, None).await;
assert_eq!(
tenths(&burst.places),
"1.2, 1.5, 1.8, 2.1, 2.5, 2.8, 3.1, 3.4, 3.8, 4.1, 4.4, 4.7"
);
assert_eq!(burst.most_registering, 1, "one registration at a time");
assert_eq!(burst.most_starts, 4, "four containers start at once");
}
#[tokio::test(start_paused = true)]
async fn a_slow_container_start_holds_back_no_other_start_of_its_batch() {
let burst = burst_into_free_slots(8, Duration::ZERO, Some(Duration::from_secs(20))).await;
let (others, slow) = burst.places.split_at(7);
assert!(
others.iter().all(|place| *place < Duration::from_secs(5)),
"the others start beside the slow one: {}",
tenths(&burst.places)
);
assert!(
slow[0] >= Duration::from_secs(21),
"the slow one takes its 20 s: {}",
tenths(&burst.places)
);
}
#[allow(clippy::print_stdout, reason = "the figures are its output")]
#[allow(
clippy::cast_precision_loss,
reason = "counts per hour, far below 2^52"
)]
async fn three_repositories() {
const JOBS: u64 = 100;
const SPACING: Duration = Duration::from_millis(31_700);
let (github, docker) = burst_fakes(Duration::ZERO);
let dir = TestDir::new("three-repositories");
let config = format!(
"{}\n[[repositories]]\nname = \"o/t\"\nprofile = \"app\"\nlabels = [\"self-hosted\", \"app\"]\nwarm = 0\n",
two_repositories().replace("slots = 2", "slots = 8")
);
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_hours(1);
burst.queue(4);
let run = tokio::spawn(engine.run());
let settled = Instant::now() + Duration::from_mins(2);
play(&mut burst, |_, now| now >= settled).await;
assert_eq!(burst.running.len(), 4, "four jobs run: {:?}", burst.seen);
burst.run = Duration::from_secs(10);
let listings = |github: &FakeGithub| {
github
.calls()
.iter()
.filter(|call| call.starts_with("queued o/r"))
.count()
};
let before = github.calls();
let reads = github.0.lock().unwrap().job_reads;
let began = Instant::now();
let mut queued = Vec::new();
let answered = std::cell::RefCell::new((listings(&github), Vec::new()));
for _ in 0..JOBS {
burst.queue(1);
queued.push(Instant::now());
let next = Instant::now() + SPACING;
play(&mut burst, |_, now| {
let mut answered = answered.borrow_mut();
let seen = listings(&github);
if seen > answered.0 {
answered.0 = seen;
answered.1.push(now);
}
now >= next
})
.await;
}
let hours = began.elapsed().as_secs_f64() / 3600.0;
let played = github.calls();
let reads = github.0.lock().unwrap().job_reads - reads;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
let answered = answered.into_inner().1;
let to_listing: Vec<Duration> = queued
.iter()
.filter_map(|queued| {
answered
.iter()
.find(|at| **at >= *queued)
.map(|at| *at - *queued)
})
.collect();
let to_container: Vec<Duration> = burst.seen.waits[4..].to_vec();
let per_hour = |prefix: &str| {
let count = |calls: &[String]| calls.iter().filter(|call| call.starts_with(prefix)).count();
(count(&played) - count(&before)) as f64 / hours
};
println!(
"three repositories, o/r with 4 running jobs and o/s, o/t idle, {JOBS} jobs 31.7 s apart, poll_interval 15s:"
);
println!(
" queued to the listing that showed it: {}",
figures(to_listing)
);
println!(" queued to container start: {}", figures(to_container));
println!(
" GitHub per hour: {:.0} o/r, {:.0} o/s and {:.0} o/t queued listings, {:.0} runner listings, {:.0} job \
reads, {:.0} registrations, {:.0} deletes",
per_hour("queued o/r"),
per_hour("queued o/s"),
per_hour("queued o/t"),
per_hour("runners "),
reads as f64 / hours,
per_hour("register "),
per_hour("delete ")
);
}
#[allow(clippy::print_stdout, reason = "the figures are its output")]
async fn the_first_job_of_an_idle_repository() {
const RUNS: u32 = 20;
let mut waits = Vec::new();
for run in 0..RUNS {
let (github, docker) = burst_fakes(Duration::ZERO);
running_job(&github, &docker, "o/r", 1, 1);
let dir = TestDir::new("idle-first-job");
let (engine, handle) = engine_with(&github, &docker, &dir, &two_repositories());
let task = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_mins(1) + Duration::from_secs(15) * run / RUNS).await;
let before = github.calls().len();
{
let mut state = github.0.lock().unwrap();
state.queued.push(job(100));
state.queued_on.insert(100, "o/s".to_owned());
}
let queued = Instant::now();
while !github.calls()[before..]
.iter()
.any(|call| call == "queued o/s")
{
tokio::time::sleep(Duration::from_millis(100)).await;
}
waits.push(queued.elapsed());
handle.shutdown(Shutdown::Force);
task.await.unwrap();
}
println!(
"an idle repository's first job, queued to the listing that showed it: {}",
figures(waits)
);
}
#[tokio::test(start_paused = true)]
#[ignore = "prints a workload's figures; run with --ignored --nocapture"]
#[allow(clippy::print_stdout, reason = "the figures are its output")]
#[allow(
clippy::cast_precision_loss,
reason = "counts per hour, far below 2^52"
)]
async fn workload_figures() {
const BURSTS: u64 = 6;
const JOBS: u64 = 40;
const PERIOD: Duration = Duration::from_mins(5);
for (followed, tick) in [
(false, None),
(true, None),
(true, Some(Duration::from_secs(5))),
] {
let (github, docker) = burst_fakes(Duration::ZERO);
if !followed {
docker.0.lock().unwrap().fail_events = u32::MAX;
}
let dir = TestDir::new("workload");
let config = CONFIG.replace("slots = 2", "slots = 8");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
let run = tokio::spawn(engine.run());
let began = Instant::now();
let mut wake = Instant::now();
for _ in 0..BURSTS {
burst.queue(JOBS);
let next = Instant::now() + PERIOD;
play(&mut burst, |_, now| {
if let Some(tick) = tick
&& now >= wake
{
handle.control.wake.notify_one();
wake = now + tick;
}
now >= next
})
.await;
}
let hours = began.elapsed().as_secs_f64() / 3600.0;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
let seen = &burst.seen;
println!(
"Docker's events {}{}:",
if followed { "followed" } else { "not followed" },
if tick.is_some() {
", a tick every 5 s"
} else {
""
}
);
println!(
" queued to container start: {}",
figures(seen.waits.clone())
);
println!(
" a slot's job ended to its next container start: {}",
figures(seen.turnovers.iter().map(|(_, took)| *took).collect())
);
let calls = github.calls();
let per_hour = |prefix: &str| {
calls.iter().filter(|call| call.starts_with(prefix)).count() as f64 / hours
};
println!(
" GitHub per hour: {:.0} queued listings, {:.0} runner listings, {:.0} job reads, {:.0} registrations, \
{:.0} deletes",
per_hour("queued "),
per_hour("runners "),
github.0.lock().unwrap().job_reads as f64 / hours,
per_hour("register "),
per_hour("delete ")
);
}
for jobs in [4, 8, 12, 16] {
println!(
"a burst of {jobs} jobs into 16 free slots, the listing that showed them to each one's container, in seconds: {}",
tenths(&staircase(jobs).await)
);
}
let contended = burst_into_free_slots(16, CONTENTION_PLACEHOLDER, None).await;
println!(
"the same burst of 16, with Docker's starts {:.2} s longer for each other one in flight, a placeholder the \
e2e replaces: {}",
CONTENTION_PLACEHOLDER.as_secs_f64(),
tenths(&contended.places)
);
three_repositories().await;
the_first_job_of_an_idle_repository().await;
}
fn stop_container(docker: &FakeDocker, id: &str) {
let mut state = docker.0.lock().unwrap();
for container in &mut state.containers {
if container.id == id {
container.running = false;
}
}
state
.events
.retain(|events| events.send(Ok(id.to_owned())).is_ok());
}
fn complete(github: &FakeGithub, job: u64, runner: &str) {
let mut state = github.0.lock().unwrap();
state.queued.retain(|queued| queued.id != job);
state.runners.retain(|registered| registered.name != runner);
let mut completed = details(job, "completed", Some("success"));
completed.runner_name = Some(runner.to_owned());
state.jobs.insert(job, completed);
}
#[tokio::test(start_paused = true)]
async fn a_job_that_ended_while_the_tick_read_it_gets_no_second_runner() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.capture_end = CaptureEnd::Container;
}
github.0.lock().unwrap().queued = vec![job(1)];
let dir = TestDir::new("closing");
let (engine, handle) = engine_in(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_secs(5)).await;
let name = registered(&github.calls())[0]
.strip_prefix("register ")
.unwrap()
.to_owned();
{
let mut state = github.0.lock().unwrap();
for runner in &mut state.runners {
if runner.name == name {
runner.busy = true;
}
}
let mut running = details(1, "in_progress", None);
running.runner_name = Some(name.clone());
state.jobs.insert(1, running);
}
docker
.0
.lock()
.unwrap()
.container_captures
.insert(name.clone(), "@@job 1 1\n".to_owned());
github.hold_jobs();
tokio::time::sleep(Duration::from_secs(15)).await;
complete(&github, 1, &name);
stop_container(&docker, &name);
tokio::time::sleep(Duration::from_secs(3)).await;
github.release_jobs();
tokio::time::sleep(Duration::from_secs(5)).await;
assert_eq!(
registered(&github.calls()).len(),
1,
"job 1 got one runner: {:?}",
github.calls()
);
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_listing_older_than_a_runners_unnamed_job_does_not_bring_the_job_back() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
running_job(&github, &docker, "o/r", 1, 1);
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(2)];
state.queued_on.insert(2, "o/r".to_owned());
let mut running = details(1, "in_progress", None);
running.runner_name = Some(state.runners[0].name.clone());
state.jobs.insert(1, running);
}
let dir = TestDir::new("generation");
let config = two_repositories().replace("slots = 2", "slots = 1");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let run = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_secs(5)).await;
github.hold_queued("o/s");
tokio::time::sleep(Duration::from_secs(12)).await;
let first = github.0.lock().unwrap().runners[0].name.clone();
complete(&github, 1, &first);
stop_container(&docker, "container-1");
tokio::time::sleep(Duration::from_secs(3)).await;
let calls = github.calls();
let started = registered(&calls);
assert_eq!(
started.len(),
1,
"a pass started a runner for job 2: {calls:?}"
);
let second = started[0].strip_prefix("register ").unwrap().to_owned();
{
let mut state = github.0.lock().unwrap();
state.queued.clear();
let mut ran = details(2, "completed", Some("success"));
ran.runner_name = Some(second.clone());
state.jobs.insert(2, ran);
state.runners.retain(|runner| runner.name != second);
}
stop_container(&docker, &second);
tokio::time::sleep(Duration::from_secs(3)).await;
github.release_queued();
tokio::time::sleep(Duration::from_secs(5)).await;
assert_eq!(
registered(&github.calls()).len(),
1,
"job 2 got one runner: {:?}",
github.calls()
);
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
#[derive(Debug, Clone, Copy)]
enum Slow {
Capture,
Removal,
}
#[derive(Debug, Clone, Copy)]
enum Ends {
Together,
SlowFirst,
}
#[tokio::test(start_paused = true)]
async fn a_slow_release_does_not_hold_back_another_slots_next_runner() {
for (slow, ends) in [
(Slow::Capture, Ends::Together),
(Slow::Removal, Ends::Together),
(Slow::Capture, Ends::SlowFirst),
(Slow::Removal, Ends::SlowFirst),
] {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
running_job(&github, &docker, "o/r", 1, 1);
running_job(&github, &docker, "o/r", 2, 2);
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(3), job(4)];
for id in [1, 2] {
let name = state.runners[usize::try_from(id - 1).unwrap()].name.clone();
let mut running = details(id, "in_progress", None);
running.runner_name = Some(name);
state.jobs.insert(id, running);
}
}
{
let mut state = docker.0.lock().unwrap();
match slow {
Slow::Capture => {
state.stuck_captures.insert("container-1".to_owned());
}
Slow::Removal => {
state
.slow_removes
.insert("container-1".to_owned(), Duration::from_mins(1));
}
}
}
let dir = TestDir::new("slow-release");
let (engine, handle) = engine_with(&github, &docker, &dir, CONFIG);
let run = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_secs(6)).await;
assert!(
registered(&github.calls()).is_empty(),
"both slots are busy"
);
for (id, slot) in [(1, 1), (2, 2)] {
let name = github.0.lock().unwrap().runners[0].name.clone();
complete(&github, id, &name);
stop_container(&docker, &format!("container-{slot}"));
if let Ends::SlowFirst = ends {
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
tokio::time::sleep(Duration::from_secs(4)).await;
let calls = github.calls();
let started: Vec<&String> = registered(&calls);
assert_eq!(
started.len(),
1,
"{slow:?}, {ends:?}: slot 2 took a job, slot 1 waits for its release: {calls:?}"
);
assert!(
started[0].contains("-app-2-"),
"{slow:?}, {ends:?}: the runner started in slot 2: {started:?}"
);
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
}
#[tokio::test(start_paused = true)]
async fn a_slot_takes_its_next_job_without_waiting_for_the_runners_delete() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
running_job(&github, &docker, "o/r", 1, 1);
running_job(&github, &docker, "o/r", 2, 2);
github.0.lock().unwrap().queued = vec![job(3), job(4)];
docker.0.lock().unwrap().container_logs.insert(
"container-1".to_owned(),
"Running job: build\nJob build completed with result: Succeeded\n".to_owned(),
);
let dir = TestDir::new("delete-off-turnover");
let config = CONFIG.replace("[host]\n", "[host]\npoll_interval = \"1h\"\n");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let run = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_secs(5)).await;
assert!(
registered(&github.calls()).is_empty(),
"both slots are busy"
);
github.hold_deletes();
let ids: Vec<u64> = github
.0
.lock()
.unwrap()
.runners
.iter()
.map(|runner| runner.id)
.collect();
for (job, slot) in [(1, 1), (2, 2)] {
let name = github.0.lock().unwrap().runners[0].name.clone();
complete(&github, job, &name);
stop_container(&docker, &format!("container-{slot}"));
}
tokio::time::sleep(Duration::from_secs(5)).await;
let calls = github.calls();
assert_eq!(
registered(&calls).len(),
2,
"both slots took a listed job: {calls:?}"
);
assert!(
!calls.contains(&format!("delete {}", ids[0])),
"the runner whose job completed is not deleted: {calls:?}"
);
assert!(
calls.contains(&format!("delete {}", ids[1])),
"the runner whose log stops before its job's end is deleted: {calls:?}"
);
github.release_deletes();
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
async fn stop_with_a_pending_delete(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> tokio::task::JoinHandle<()> {
running_job(github, docker, "o/r", 1, 1);
let config = CONFIG.replace("[host]\n", "[host]\npoll_interval = \"1h\"\n");
let (engine, handle) = engine_with(github, docker, dir, &config);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github.hold_deletes();
let runner = github.0.lock().unwrap().runners[0].clone();
complete(github, 1, &runner.name);
stop_container(docker, "container-1");
tokio::time::sleep(Duration::from_secs(5)).await;
assert!(
github.calls().contains(&format!("delete {}", runner.id)),
"the released runner's delete is sent: {:?}",
github.calls()
);
handle.shutdown(Shutdown::Graceful);
run
}
#[tokio::test(start_paused = true)]
async fn a_stop_waits_for_a_released_runners_delete() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
let dir = TestDir::new("stop-waits-for-delete");
let run = stop_with_a_pending_delete(&github, &docker, &dir).await;
tokio::time::sleep(Duration::from_secs(10)).await;
assert!(!run.is_finished(), "the stop waits for the delete");
github.release_deletes();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.expect("the stop ends once GitHub answers")
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_stop_waits_for_a_delete_github_does_not_answer_only_until_its_deadline() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
let dir = TestDir::new("stop-delete-deadline");
let run = stop_with_a_pending_delete(&github, &docker, &dir).await;
tokio::time::timeout(STOP_DEADLINE + Duration::from_secs(10), run)
.await
.expect("the stop ends by its deadline")
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_runner_whose_capture_named_its_job_shows_running_before_the_next_poll() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
docker.0.lock().unwrap().captures_wait = Some(github.clone());
github.0.lock().unwrap().queued = vec![job(1)];
let dir = TestDir::new("named-job-status");
let config = CONFIG.replace("[host]\n", "[host]\npoll_interval = \"1h\"\n");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let run = tokio::spawn(engine.run());
let state = |handle: &Handle| {
let status = handle.status.borrow();
api::Status::slot_state(&status.profiles[0], &status.profiles[0].slots[0])
};
tokio::time::sleep(Duration::from_secs(5)).await;
assert_eq!(state(&handle).word(), "starting", "{:?}", docker.calls());
let name = registered(&github.calls())[0]
.strip_prefix("register ")
.unwrap()
.to_owned();
docker
.0
.lock()
.unwrap()
.container_captures
.insert(name, "@@job 1 1\n".to_owned());
tokio::time::sleep(Duration::from_secs(1)).await;
assert_eq!(state(&handle).word(), "running");
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_repository_removed_and_added_back_keeps_what_its_runners_took() {
let (github, docker) = (FakeGithub::default(), FakeDocker::default());
docker.0.lock().unwrap().captures_wait = Some(github.clone());
running_job(&github, &docker, "o/r", 1, 1);
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(2)];
state.queued_on.insert(2, "o/r".to_owned());
let mut running = details(1, "in_progress", None);
running.runner_name = Some(state.runners[0].name.clone());
state.jobs.insert(1, running);
}
let dir = TestDir::new("readded");
let config = two_repositories().replace("slots = 2", "slots = 1");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let run = tokio::spawn(engine.run());
tokio::time::sleep(Duration::from_secs(5)).await;
github.hold_queued("o/s");
tokio::time::sleep(Duration::from_secs(12)).await;
let first = github.0.lock().unwrap().runners[0].name.clone();
complete(&github, 1, &first);
stop_container(&docker, "container-1");
tokio::time::sleep(Duration::from_secs(3)).await;
let calls = github.calls();
let started = registered(&calls);
assert_eq!(
started.len(),
1,
"a pass started a runner for job 2: {calls:?}"
);
let second = started[0].strip_prefix("register ").unwrap().to_owned();
docker
.0
.lock()
.unwrap()
.container_captures
.insert(second.clone(), "@@job 2 2\n".to_owned());
tokio::time::sleep(Duration::from_secs(1)).await;
complete(&github, 2, &second);
stop_container(&docker, &second);
tokio::time::sleep(Duration::from_secs(3)).await;
for config in [
config.replace("name = \"o/r\"", "name = \"o/t\""),
config.clone(),
] {
handle
.control
.replace_config(
Config::parse(&config).unwrap(),
None,
None,
Commit::new(|| Ok(None)),
RunningSlots::Keep,
Author::Owner,
)
.await
.unwrap();
}
github.0.lock().unwrap().queued = vec![job(2)];
github.release_queued();
tokio::time::sleep(Duration::from_secs(5)).await;
assert_eq!(
registered(&github.calls()).len(),
1,
"job 2 got one runner: {:?}",
github.calls()
);
handle.shutdown(Shutdown::Force);
run.await.unwrap();
}
fn read_rounds(reads: &mut Vec<Instant>, count: usize, seen: &mut usize) {
if count > *seen {
if reads
.last()
.is_none_or(|&at| Instant::now() - at > Duration::from_secs(5))
{
reads.push(Instant::now());
}
*seen = count;
}
}
fn gaps_of(times: &[Instant]) -> Vec<Duration> {
times.windows(2).map(|pair| pair[1] - pair[0]).collect()
}
#[tokio::test(start_paused = true)]
async fn job_reads_slow_first_then_listings_and_a_start_still_goes_out() {
let (github, docker) = burst_fakes(Duration::ZERO);
let dir = TestDir::new("governor-pace");
let config = two_repositories().replace("slots = 2", "slots = 5");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_hours(1);
burst.queue(4);
let run = tokio::spawn(engine.run());
let settled = Instant::now() + Duration::from_mins(2);
play(&mut burst, |_, now| now >= settled).await;
assert_eq!(burst.running.len(), 4, "four jobs run: {:?}", burst.seen);
github.limit(1000, 25);
let began = Instant::now();
let (mut reads, mut seen) = (Vec::new(), github.0.lock().unwrap().job_reads);
let until = began + Duration::from_mins(14);
while Instant::now() < until {
burst.step(Instant::now());
let count = github.0.lock().unwrap().job_reads;
read_rounds(&mut reads, count, &mut seen);
tokio::time::sleep(Duration::from_millis(100)).await;
}
let listings = burst.listed_since("o/r", began);
let idle = gaps_of(&burst.listed_since("o/s", began));
let (read_gaps, listing_gaps) = (gaps_of(&reads), gaps_of(&listings));
let early = |times: &[Instant], gaps: &[Duration]| {
times
.windows(2)
.zip(gaps)
.take_while(|(pair, _)| pair[1] - began < Duration::from_mins(2))
.map(|(_, gap)| *gap)
.collect::<Vec<_>>()
};
for gap in early(&reads, &read_gaps) {
assert!(gap <= Duration::from_secs(17), "early read gap {gap:?}");
}
for gap in early(&listings, &listing_gaps) {
assert!(gap <= Duration::from_secs(6), "early listing gap {gap:?}");
}
let first_slow = |times: &[Instant], gaps: &[Duration], least: u64| {
times
.windows(2)
.zip(gaps)
.find(|(_, gap)| **gap >= Duration::from_secs(least))
.map(|(pair, _)| pair[1])
};
let slow_reads = first_slow(&reads, &read_gaps, 28).expect("the reads slowed");
let held = first_slow(&listings, &listing_gaps, 14).expect("o/r's listings slowed");
let slow_listings = first_slow(&listings, &listing_gaps, 28).expect("the listings slowed");
assert!(
slow_reads + Duration::from_mins(3) < held && held < slow_listings,
"reads slowed at {:?}, o/r's listings to the idle pace at {:?}, every listing at {:?} after the limit",
slow_reads - began,
held - began,
slow_listings - began
);
assert!(
read_gaps.last().unwrap() >= &Duration::from_secs(58)
&& listing_gaps.last().unwrap() >= &Duration::from_secs(58)
&& idle.last().unwrap() >= &Duration::from_secs(58),
"reads {read_gaps:?}, o/r's listings {listing_gaps:?}, o/s's {idle:?}"
);
let calls = github.calls().len();
burst.queue(1);
let queued = Instant::now();
let (mut listed, mut registered) = (None, None);
while registered.is_none() && Instant::now() < queued + Duration::from_mins(3) {
burst.step(Instant::now());
for call in &github.calls()[calls..] {
if call.starts_with("queued o/r") {
listed.get_or_insert(Instant::now());
}
if call.starts_with("register ") {
registered.get_or_insert(Instant::now());
}
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
handle.shutdown(Shutdown::Force);
run.await.unwrap();
let (listed, registered) = (listed.unwrap(), registered.expect("the job got a runner"));
assert!(
registered - listed <= Duration::from_secs(3),
"listed {:?} after it was queued, registered {:?} after that",
listed - queued,
registered - listed
);
}
#[tokio::test(start_paused = true)]
async fn a_healthy_limit_leaves_the_engines_calls_as_they_were() {
async fn trace(limit: bool) -> Vec<String> {
let (github, docker) = burst_fakes(Duration::ZERO);
let dir = TestDir::new("governor-trace");
let config = CONFIG.replace("slots = 2", "slots = 4");
let (engine, handle) = engine_with(&github, &docker, &dir, &config);
let mut burst = Burst::new(&github, &docker, u64::MAX);
burst.run = Duration::from_mins(4);
if limit {
github.limit(5000, 0);
}
burst.queue(4);
let run = tokio::spawn(engine.run());
let until = Instant::now() + Duration::from_mins(10);
play(&mut burst, |burst, now| {
now >= until || (burst.done() && now + Duration::from_mins(8) >= until)
})
.await;
handle.shutdown(Shutdown::Force);
run.await.unwrap();
let trace = github.0.lock().unwrap().trace.clone();
trace
.into_iter()
.map(|call| match call.strip_prefix("register ") {
Some(name) => format!(
"register {}",
name.rsplit_once('-').map_or(name, |(kept, _)| kept)
),
None => call,
})
.collect()
}
let (without, with) = (trace(false).await, trace(true).await);
assert!(without.len() > 100, "a trace to compare: {}", without.len());
assert_eq!(without, with);
}