use super::*;
use crate::scheduler::IDLE_TIMEOUT;
async fn idle_runner_beside_a_queued_job(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
idle_runner(github, docker);
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.capture_end = CaptureEnd::Container;
}
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(5)];
state.queued_on.insert(5, "o/s".to_owned());
}
let (mut engine, handle) = engine_with(github, docker, dir, &two_repositories());
engine.recover().await.unwrap();
for occupant in engine.slots.values_mut().flatten().flatten() {
occupant.started_at -= IDLE_TIMEOUT;
}
(engine, handle)
}
#[tokio::test(start_paused = true)]
async fn removing_an_idle_runner_does_not_hold_back_a_start_for_another_repository() {
let github = FakeGithub::default();
let docker = FakeDocker::default();
let dir = TestDir::new("pacing");
let (mut engine, handle) = idle_runner_beside_a_queued_job(&github, &docker, &dir).await;
let began = tokio::time::Instant::now();
engine.tick(Shutdown::No).await;
let took = began.elapsed();
assert_eq!(removed(&docker.calls()), ["remove container-1"]);
assert_eq!(registered(&github.calls()).len(), 1, "o/s got its runner");
assert!(
took < super::super::history::CAPTURE_DRAIN,
"the tick took {took:?}"
);
handle.store.flush().await;
let history = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
let runners: Vec<(String, Option<String>, Option<String>)> = history
.prepare("SELECT repository, purpose, stopped_for FROM runners ORDER BY started_at")
.unwrap()
.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
.unwrap()
.collect::<Result<_, _>>()
.unwrap();
assert_eq!(
runners,
[
("o/r".to_owned(), None, Some("expired".to_owned())),
("o/s".to_owned(), Some("job".to_owned()), None)
]
);
}
#[tokio::test(start_paused = true)]
#[ignore = "prints a workload's figures; run with --ignored --nocapture"]
#[allow(clippy::print_stdout, reason = "the figures are its output")]
async fn workload_figures() {
const PERIOD: Duration = Duration::from_secs(40);
const RUN: Duration = Duration::from_secs(25);
const TAKEN_AFTER: Duration = Duration::from_secs(20);
const LENGTH: Duration = Duration::from_mins(12);
const POLL: Duration = Duration::from_secs(15);
let github = FakeGithub::default();
let docker = FakeDocker::default();
let dir = TestDir::new("workload");
{
let mut state = github.0.lock().unwrap();
state.strict = true;
state.latency = Duration::from_millis(300);
}
{
let mut state = docker.0.lock().unwrap();
state.captures_wait = Some(github.clone());
state.capture_end = CaptureEnd::Container;
state.latency = DockerLatency {
start: Duration::from_millis(1000),
remove_running: Duration::from_millis(1000),
};
}
let config = two_repositories().replace("slots = 2", "slots = 8");
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.recover().await.unwrap();
let began = tokio::time::Instant::now();
let waits: Arc<Mutex<Vec<Duration>>> = Arc::default();
let world = tokio::spawn(world(
github.clone(),
docker.clone(),
waits.clone(),
began,
[PERIOD, RUN, TAKEN_AFTER, LENGTH],
));
while began.elapsed() < LENGTH {
engine.tick(Shutdown::No).await;
for occupant in engine.slots.values_mut().flatten().flatten() {
if occupant.repository == "o/r" && occupant.job.is_none() && !occupant.busy {
occupant.started_at = std::time::SystemTime::now() - IDLE_TIMEOUT;
}
}
std::thread::sleep(Duration::from_millis(1001));
tokio::time::sleep(POLL).await;
}
world.abort();
drop(handle);
let mut waits = waits.lock().unwrap().clone();
waits.sort();
let at =
|percent: usize| waits[(waits.len() * percent / 100).min(waits.len() - 1)].as_secs_f64();
let calls = github.calls();
let count = |prefix: &str| calls.iter().filter(|call| call.starts_with(prefix)).count();
println!(
"o/s jobs {}: queued to container start p50 {:.1}s, p90 {:.1}s, max {:.1}s",
waits.len(),
at(50),
at(90),
at(100)
);
println!(
"GitHub: {} queued listings, {} runner listings, {} registrations, {} deletes",
count("queued "),
count("runners "),
count("register "),
count("delete ")
);
}
async fn world(
github: FakeGithub,
docker: FakeDocker,
waits: Arc<Mutex<Vec<Duration>>>,
began: tokio::time::Instant,
[period, run, taken_after, length]: [Duration; 4],
) {
let mut world = World {
next_id: 1,
next_burst: began,
queued_at: HashMap::new(),
taken_elsewhere: Vec::new(),
ends: Vec::new(),
assigned: BTreeSet::new(),
};
loop {
let now = tokio::time::Instant::now();
if now >= world.next_burst && now - began < length {
world.burst(&github, now, taken_after);
world.next_burst += period;
}
world.step(&github, &docker, &waits, now, run);
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
struct World {
next_id: u64,
next_burst: tokio::time::Instant,
queued_at: HashMap<u64, tokio::time::Instant>,
taken_elsewhere: Vec<(tokio::time::Instant, u64)>,
ends: Vec<(tokio::time::Instant, String, String)>,
assigned: BTreeSet<String>,
}
impl World {
fn burst(&mut self, github: &FakeGithub, now: tokio::time::Instant, taken_after: Duration) {
let mut state = github.0.lock().unwrap();
for repository in ["o/s", "o/s", "o/s", "o/r", "o/r", "o/r"] {
let id = self.next_id;
self.next_id += 1;
state.queued.push(job(id));
state.queued_on.insert(id, repository.to_owned());
if repository == "o/r" {
self.taken_elsewhere.push((now + taken_after, id));
} else {
self.queued_at.insert(id, now);
}
}
}
fn step(
&mut self,
github: &FakeGithub,
docker: &FakeDocker,
waits: &Mutex<Vec<Duration>>,
now: tokio::time::Instant,
run: Duration,
) {
let mut docker_state = docker.0.lock().unwrap();
let mut github_state = github.0.lock().unwrap();
let gone: Vec<u64> = self
.taken_elsewhere
.iter()
.filter(|(at, _)| *at <= now)
.map(|(_, id)| *id)
.collect();
github_state.queued.retain(|job| !gone.contains(&job.id));
self.taken_elsewhere.retain(|(at, _)| *at > now);
let up: Vec<(String, String)> = docker_state
.containers
.iter()
.filter(|container| {
container.running
&& container
.labels
.get("runnerd.repository")
.map(String::as_str)
== Some("o/s")
&& !self.assigned.contains(&container.id)
})
.map(|container| (container.id.clone(), container.name.clone()))
.collect();
for (id, name) in up {
let Some(job) = github_state
.queued
.iter()
.find(|job| github_state.queued_on.get(&job.id).map(String::as_str) == Some("o/s"))
.map(|job| job.id)
else {
break;
};
github_state.queued.retain(|queued| queued.id != job);
for runner in &mut github_state.runners {
if runner.name == name {
runner.busy = true;
}
}
let mut running = details(job, "in_progress", None);
running.runner_name = Some(name.clone());
github_state.jobs.insert(job, running);
docker_state
.container_captures
.insert(id.clone(), format!("@@job {job} {job}\n"));
let started = docker_state
.started_on_clock
.get(&name)
.copied()
.unwrap_or(now);
waits
.lock()
.unwrap()
.push(started.saturating_duration_since(self.queued_at[&job]));
self.assigned.insert(id.clone());
self.ends.push((now + run, id, name));
}
for (_, id, name) in self.ends.iter().filter(|(at, _, _)| *at <= now) {
if let Some(container) = docker_state
.containers
.iter_mut()
.find(|container| container.id == *id)
{
container.running = false;
}
github_state.runners.retain(|runner| runner.name != *name);
}
self.ends.retain(|(at, _, _)| *at > now);
}
}