use super::*;
use crate::admission::pooled::POOL_AFTER;
use crate::api::{AdmissionPolicy, Basis, JobPeak};
use crate::observe::metrics::Events;
use crate::scheduler::PRIORITY_AFTER;
use crate::scheduler::{Decision, Key, Plan, SlotRef, StartFor};
use crate::store::{RunnerRecord, SampleRecord};
use crate::time;
const G: u64 = 1 << 30;
const M: u64 = 1 << 20;
pub(super) fn measured() -> String {
CONFIG
.replace("slots = 2", "slots = 4")
.replace("admission = \"caps\"", "admission = \"measured\"")
}
async fn history(store: &Store, peak: u64, oom_kills: u64) {
history_of(store, "build", 1000, peak, oom_kills).await;
}
pub(super) async fn history_of(store: &Store, name: &str, first: u64, peak: u64, oom_kills: u64) {
runs_of(store, name, first, 10, peak, oom_kills, Ended::Lost).await;
}
#[derive(Clone, Copy, PartialEq)]
enum Ended {
Lost,
Succeeded,
}
async fn runs_of(
store: &Store,
name: &str,
first: u64,
count: u64,
peak: u64,
oom_kills: u64,
ended: Ended,
) {
let at = time::now_ms() - 3600 * 1000;
for i in 0..count {
let id = first + i;
let runner = format!("earlier-{id}");
store.queued(
"o/r",
&QueuedJob {
created_at: at,
name: name.to_owned(),
..job(id)
},
time::now_ms(),
);
store.runner_started(RunnerRecord {
purpose: None,
name: runner.clone(),
repository: "o/r".to_owned(),
profile: "app".to_owned(),
slot: 1,
container: format!("c-{id}"),
started_at: at,
memory_cap: Some(4 * G),
cpu_cap: Some(2000),
admission: None,
});
store.claimed(id, 1, &runner, at + 1000);
store.samples(vec![SampleRecord {
runner: runner.clone(),
at: at + 2000,
cpu: 0,
cpu_usage: 0,
memory: peak,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
written: None,
dirty: None,
events: Events {
oom: Some(0),
oom_kill: Some(if i == count - 1 { oom_kills } else { 0 }),
sock_throttled: None,
local_oom: None,
},
}]);
match ended {
Ended::Lost => store.lost(id, at + 60_000),
Ended::Succeeded => store.details(JobDetails {
id,
run_id: 1,
run_attempt: 1,
name: name.to_owned(),
workflow_name: None,
head_branch: None,
html_url: None,
labels: Vec::new(),
created_at: None,
status: "completed".to_owned(),
conclusion: Some("success".to_owned()),
started_at: None,
completed_at: None,
runner_name: Some(runner.clone()),
steps: Vec::new(),
}),
}
store.runner_removed(&runner, at + 60_000);
}
store.flush().await;
}
pub(super) fn slice_at(store: &Store, memory: u64) {
store.host_sample(api::HostSample {
at: time::now_ms() / 1000,
cpu: 0,
memory,
});
}
async fn measured_engine(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
peak: u64,
oom_kills: u64,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let (mut engine, handle) = engine_with(github, docker, dir, &measured());
engine.memory_max = Some(10 * G);
history(&handle.store, peak, oom_kills).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
(engine, handle)
}
#[tokio::test]
async fn small_predicted_peaks_start_more_runners_than_the_caps_fit() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 4 * 1280 * M);
assert_eq!(
handle.status.borrow().budget.admission,
AdmissionPolicy::Measured
);
}
#[tokio::test]
async fn on_a_fresh_host_the_default_admission_reserves_caps_until_jobs_have_a_history() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let text = CONFIG
.replace("admission = \"caps\"\n", "")
.replace("slots = 2", "slots = 4");
assert!(!text.contains("admission"), "the config leaves it out");
let (mut engine, handle) = engine_with(&github, &docker, &dir, &text);
engine.memory_max = Some(10 * G);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"two 4G caps fit the 10G slice"
);
engine.publish();
let budget = handle.status.borrow().budget.clone();
assert_eq!(budget.admission, AdmissionPolicy::Measured);
assert_eq!(budget.memory_in_use, 8 * G, "each at its cap");
assert!(
budget.memory_in_use <= 10 * G,
"nothing starts over MemoryMax"
);
}
#[tokio::test]
async fn a_reload_that_drops_the_admission_key_keeps_the_running_runners_reservations() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4, "1.25G each");
engine.publish();
let before = handle.status.borrow().budget.memory_in_use;
let unset = measured().replace("admission = \"measured\"\n", "");
assert!(!unset.contains("admission"), "the key is gone");
engine.reload(Config::parse(&unset).unwrap(), Some(10 * G), None);
engine.publish();
let budget = handle.status.borrow().budget.clone();
assert_eq!(budget.admission, AdmissionPolicy::Measured, "the default");
assert_eq!(
budget.memory_in_use, before,
"each runner keeps what it reserved, not its cap"
);
}
#[tokio::test]
async fn a_history_read_less_than_a_minute_ago_stands() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
engine.recover().await.unwrap();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
history(&handle.store, G, 0).await;
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"the jobs reserve their 4G caps until the history is read again"
);
}
#[tokio::test]
async fn nothing_starts_while_the_slice_is_at_the_high_water_mark() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 92 * G / 10);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 0);
engine.publish();
assert!(
handle.status.borrow().budget.held,
"the budget says admission is held"
);
slice_at(&handle.store, 3 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4);
engine.publish();
assert!(!handle.status.borrow().budget.held, "admission resumed");
}
#[tokio::test]
async fn a_reseed_waits_while_the_slice_is_at_the_high_water_mark() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
engine.recover().await.unwrap();
engine.request_reseed("app").unwrap();
slice_at(&handle.store, 92 * G / 10);
engine.tick(Shutdown::No).await;
assert!(engine.no_reseeds(), "the seed waits");
slice_at(&handle.store, 3 * G);
engine.tick(Shutdown::No).await;
assert!(engine.reseeding("app"), "the seed starts");
}
#[tokio::test]
async fn without_a_recent_sample_of_the_slice_runners_are_admitted_at_their_caps() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
handle.store.host_sample(api::HostSample {
at: time::now_ms() / 1000 - 60,
cpu: 0,
memory: 2 * G,
});
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "2 × 4G fit in 10G");
}
#[tokio::test]
async fn peaks_sampled_with_page_cache_do_not_predict_what_a_job_reserves() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
history(&handle.store, G, 0).await;
rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.execute("UPDATE runners SET memory_measure = NULL", [])
.unwrap();
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"no run measured the new way: 2 × the 4G cap"
);
}
#[tokio::test]
async fn a_kill_for_memory_from_before_the_upgrade_keeps_its_job_on_its_cap() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
history(&handle.store, G, 1).await;
rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.execute("UPDATE runners SET memory_measure = NULL", [])
.unwrap();
history_of(&handle.store, "lint", 2000, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"builds reserve their 4G cap, not the lints' 1.25G"
);
let mut reserved: Vec<String> = Vec::new();
for _ in 0..200 {
handle.store.flush().await;
reserved = rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.prepare("SELECT admission FROM runners WHERE name NOT LIKE 'earlier-%'")
.unwrap()
.query_map([], |row| row.get(0))
.unwrap()
.collect::<Result<_, _>>()
.unwrap();
if reserved.len() == 2 {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
let expires =
(time::now_ms() - 3600 * 1000 + 60_000 + crate::admission::UNPREDICTABLE_FOR) / 1000;
assert_eq!(reserved.len(), 2);
for admission in reserved {
let admission: api::Admission = serde_json::from_str(&admission).unwrap();
assert!(
matches!(admission.basis, Basis::Unpredictable { until } if until.abs_diff(expires) <= 1),
"{:?}",
admission.basis
);
}
}
#[tokio::test]
async fn a_job_killed_for_memory_is_admitted_at_its_cap() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 1).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"its peaks say nothing for a week"
);
}
#[tokio::test]
async fn a_jobs_record_says_how_its_runner_was_admitted() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
let runner = registered(&github.calls())[0]
.trim_start_matches("register ")
.to_owned();
handle.store.claimed(1, 1, &runner, time::now_ms());
handle.store.flush().await;
let admission = handle.store.job(1).await.unwrap().unwrap().admission;
assert_eq!(
admission,
Some(api::Admission {
policy: AdmissionPolicy::Measured,
reserved: 1280 * M,
basis: Basis::Job {
runs: 10,
peak: JobPeak::P95
},
free: Some(10 * G),
slice: Some(2 * G),
cpu: None,
})
);
}
pub(super) fn runners_use(github: &FakeGithub, store: &Store, memory: u64) {
runners_used(github, store, memory, time::now_ms());
}
fn runners_used(github: &FakeGithub, store: &Store, memory: u64, at: u64) {
let samples = registered(&github.calls())
.iter()
.map(|call| SampleRecord {
runner: call.trim_start_matches("register ").to_owned(),
at,
cpu: 0,
cpu_usage: 0,
memory,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
written: None,
dirty: None,
events: Events::default(),
})
.collect();
store.samples(samples);
}
#[tokio::test]
async fn a_runner_using_more_than_it_reserved_counts_at_what_it_uses() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
runners_use(&github, &handle.store, 4 * G);
slice_at(&handle.store, 8 * G);
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
3,
"one more 1.25G runner fits in the 2G left, not two"
);
engine.publish();
assert_eq!(
handle.status.borrow().budget.memory_in_use,
8 * G + 1280 * M
);
}
#[tokio::test]
async fn back_under_caps_every_runner_counts_at_its_cap_whatever_it_reserved() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "2 × 1.25G reserved");
for runner in &mut github.0.lock().unwrap().runners {
runner.busy = true;
}
let caps = CONFIG
.replace("slots = 2", "slots = 3")
.replace("memory = \"4G\"", "memory = \"3G\"");
engine.reload(Config::parse(&caps).unwrap(), Some(10 * G), None);
github.0.lock().unwrap().queued = (1..=3).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"8G of caps run: a 3G runner does not fit"
);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 8 * G);
}
#[tokio::test]
async fn once_its_job_is_known_a_runner_reserves_that_jobs_prediction() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
handle.store.queued(
"o/r",
&QueuedJob {
name: "e2e".to_owned(),
..job(9)
},
time::now_ms(),
);
docker
.0
.lock()
.unwrap()
.captures
.insert(("1".to_owned(), "0".to_owned()), "@@job 9 1\n".to_owned());
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
until_state(&handle, 9, JobState::Running).await;
github.0.lock().unwrap().queued = Vec::new();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 3584 * M);
handle.store.flush().await;
let admission = handle.store.job(9).await.unwrap().unwrap().admission;
assert_eq!(
admission.map(|admission| (admission.reserved, admission.basis)),
Some((
3584 * M,
Basis::Job {
runs: 10,
peak: JobPeak::P95
}
))
);
}
#[tokio::test]
async fn a_runner_that_takes_a_job_no_listing_saw_reserves_that_jobs_prediction_once_github_names_it()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, 3 * G, 0).await;
history_of(&handle.store, "lint", 2000, G, 0).await;
docker
.0
.lock()
.unwrap()
.captures
.insert(("1".to_owned(), "0".to_owned()), "@@job 9 1\n".to_owned());
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
until_state(&handle, 9, JobState::Running).await;
github.0.lock().unwrap().queued = Vec::new();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
let mut lint = details(9, "in_progress", None);
lint.name = "lint".to_owned();
lint.workflow_name = None;
github.0.lock().unwrap().jobs.insert(9, lint);
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 1280 * M);
}
#[tokio::test]
async fn an_idle_runner_reserves_the_largest_prediction_of_the_jobs_queued_since() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1, "1.25G for the build");
github.0.lock().unwrap().queued = vec![
job(1),
QueuedJob {
name: "e2e".to_owned(),
..job(2)
},
];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
engine.publish();
assert_eq!(
handle.status.borrow().budget.memory_in_use,
2 * 3584 * M,
"both runners reserve the e2e's 3.5G"
);
}
#[tokio::test]
async fn waiting_runners_raised_past_the_budget_for_a_heavier_job_are_stopped_until_the_rest_fits()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
4,
"4 × 1.25G for the builds"
);
github.0.lock().unwrap().queued.push(QueuedJob {
name: "e2e".to_owned(),
..job(5)
});
engine.tick(Shutdown::No).await;
let deleted: BTreeSet<String> = github
.calls()
.into_iter()
.filter(|call| call.starts_with("delete "))
.collect();
assert_eq!(deleted.len(), 2, "two stop, so the other two's 7G fits");
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 2 * 3584 * M);
let starting = handle
.status
.borrow()
.queued
.iter()
.filter(|job| job.waiting == Some(api::Waiting::RunnerStarting))
.count();
assert_eq!(
starting, 2,
"only the two runners left say they start a job"
);
}
#[tokio::test]
async fn running_runners_count_at_their_caps_once_the_slice_has_no_recent_sample() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
runners_use(&github, &handle.store, G);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 2 * 1280 * M);
handle.store.host_sample(api::HostSample {
at: time::now_ms() / 1000 - 60,
cpu: 0,
memory: 2 * G,
});
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 2 * 4 * G);
}
#[tokio::test]
async fn a_runner_whose_own_sample_is_old_counts_at_its_cap() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
runners_used(&github, &handle.store, G, time::now_ms() - 60_000);
slice_at(&handle.store, 2 * G);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 4 * G);
}
#[tokio::test]
async fn a_runner_whose_container_is_gone_counts_nothing_while_it_keeps_its_slot() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
runners_used(&github, &handle.store, G, time::now_ms() - 60_000);
slice_at(&handle.store, 2 * G);
let runner = engine
.occupied()
.map(|(_, occupant)| occupant.runner_name.clone())
.next()
.unwrap();
engine.publish();
let counted = handle.status.borrow().budget.memory_in_use;
handle.store.containers_gone(vec![runner.clone()]);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 0);
handle.store.containers_read(&[runner]);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, counted);
}
#[tokio::test]
async fn two_light_runs_then_a_burst_of_heavy_copies_with_warm_up_never_overlap_past_memory_max() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
runs_of(
&handle.store,
"build",
1000,
2,
200 * M,
0,
Ended::Succeeded,
)
.await;
engine.recover().await.unwrap();
docker.0.lock().unwrap().slot_captures =
HashMap::from([(1, "@@job 1 1\n".to_owned()), (2, "@@job 2 1\n".to_owned())]);
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
until_state(&handle, 1, JobState::Running).await;
until_state(&handle, 2, JobState::Running).await;
github.0.lock().unwrap().queued = Vec::new();
slice_at(&handle.store, G);
engine.tick(Shutdown::No).await;
{
let mut state = github.0.lock().unwrap();
for runner in &mut state.runners {
(runner.status, runner.busy) = ("online".to_owned(), true);
}
state.queued = (3..=4).map(job).collect();
}
runners_use(&github, &handle.store, 300 * M);
slice_at(&handle.store, G);
engine.tick(Shutdown::No).await;
let started = registered(&github.calls()).len() as u64;
assert_eq!(started, 3);
engine.publish();
assert_eq!(
handle.status.borrow().budget.memory_in_use,
300 * M + 2 * 3584 * M
);
assert!(
started * 3 * G <= 10 * G,
"{started} copies at 3G fit the 10G slice"
);
}
#[tokio::test]
async fn copies_arriving_one_by_one_beside_busy_runners_whose_job_is_unknown_take_their_profiles_p95()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
runs_of(
&handle.store,
"build",
1000,
2,
200 * M,
0,
Ended::Succeeded,
)
.await;
engine.recover().await.unwrap();
docker.0.lock().unwrap().fail_execs = u32::MAX;
let mut started = Vec::new();
for copy in 1..=4 {
{
let mut state = github.0.lock().unwrap();
for runner in &mut state.runners {
(runner.status, runner.busy) = ("online".to_owned(), true);
}
state.queued.clear();
}
engine.tick(Shutdown::No).await;
github.0.lock().unwrap().queued = vec![job(copy)];
runners_use(&github, &handle.store, 300 * M);
slice_at(&handle.store, G);
engine.tick(Shutdown::No).await;
started.push(registered(&github.calls()).len());
}
assert_eq!(started, [1, 2, 2, 2]);
engine.publish();
assert_eq!(handle.status.borrow().budget.memory_in_use, 2 * 3584 * M);
}
#[tokio::test]
async fn jobs_wait_for_cpu_the_budget_lacks_and_their_records_say_what_they_reserved_and_waited() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = measured().replace(
"admission = \"measured\"",
"admission = \"measured\"\ncpu_overcommit = 1.0",
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(10 * G);
engine.cpus = Some(4000);
history(&handle.store, G, 0).await;
rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.execute("UPDATE jobs SET cpu_average = 1500", [])
.unwrap();
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
let first = registered(&github.calls())
.iter()
.map(|call| call.trim_start_matches("register ").to_owned())
.collect::<Vec<_>>();
assert_eq!(first.len(), 2);
engine.publish();
let cpu = handle.status.borrow().budget.cpu;
assert_eq!(
cpu,
Some(api::CpuBudget {
cpus: 4000,
budget: 4000,
in_use: 3000,
held: true,
})
);
handle.store.claimed(1, 1, &first[0], time::now_ms());
handle.store.flush().await;
let admission = handle.store.job(1).await.unwrap().unwrap().admission;
assert_eq!(
admission.and_then(|admission| admission.cpu),
Some(api::CpuAdmission {
reserved: 1500,
basis: Basis::Job {
runs: 10,
peak: JobPeak::P95,
},
free: 4000,
waited: None,
})
);
engine.cpus = Some(8000);
engine.tick(Shutdown::No).await;
let all = registered(&github.calls())
.iter()
.map(|call| call.trim_start_matches("register ").to_owned())
.collect::<Vec<_>>();
assert_eq!(all.len(), 4);
handle.store.claimed(3, 1, &all[2], time::now_ms());
handle.store.flush().await;
let admission = handle.store.job(3).await.unwrap().unwrap().admission;
assert!(
admission
.and_then(|admission| admission.cpu)
.is_some_and(|cpu| cpu.waited.is_some()),
"the third build's record says it waited for CPU: {admission:?}"
);
engine.publish();
assert!(
handle
.status
.borrow()
.budget
.cpu
.is_some_and(|cpu| !cpu.held),
"nothing waits for CPU any more"
);
}
#[tokio::test]
async fn a_job_with_only_runs_from_before_the_upgrade_reserves_its_profiles_largest_peak_with_its_cpu_recorded()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
history(&handle.store, G, 0).await;
history_of(&handle.store, "legacy", 2000, 3 * G, 0).await;
let db = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
db.execute(
"UPDATE runners SET memory_measure = NULL WHERE name LIKE 'earlier-2%'",
[],
)
.unwrap();
db.execute("UPDATE jobs SET cpu_average = 1500", [])
.unwrap();
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![QueuedJob {
name: "legacy".to_owned(),
..job(1)
}];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
let runner = registered(&github.calls())[0]
.trim_start_matches("register ")
.to_owned();
handle.store.claimed(1, 1, &runner, time::now_ms());
handle.store.flush().await;
let admission = handle
.store
.job(1)
.await
.unwrap()
.unwrap()
.admission
.unwrap();
assert_eq!(
(admission.reserved, admission.basis, admission.cpu),
(1280 * M, Basis::ProfileMax { runs: 10 }, None)
);
}
fn cpu_measured() -> String {
measured().replace("slots = 4", "slots = 6").replace(
"admission = \"measured\"",
"admission = \"measured\"\ncpu_overcommit = 1.0",
)
}
#[tokio::test]
async fn after_a_reload_to_caps_running_runners_count_at_their_cpu_limits() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &cpu_measured());
engine.cpus = Some(4000);
history(&handle.store, G, 0).await;
rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.execute("UPDATE jobs SET cpu_average = 500", [])
.unwrap();
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4, "4 × 0.5 CPU fit 4");
let caps = cpu_measured().replace("admission = \"measured\"", "admission = \"caps\"");
engine.reload(Config::parse(&caps).unwrap(), None, Some(4000));
engine.publish();
assert_eq!(
handle.status.borrow().budget.cpu.map(|cpu| cpu.in_use),
Some(8000)
);
github.0.lock().unwrap().queued = (1..=6).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
4,
"no 2-CPU runner fits a budget the running ones fill"
);
}
#[tokio::test]
async fn a_reload_that_sets_cpu_overcommit_admits_by_the_slices_cpus_it_read() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, CONFIG);
engine.recover().await.unwrap();
engine.publish();
assert_eq!(handle.status.borrow().budget.cpu, None);
let with_budget = CONFIG.replace(
"isolation = \"sysbox\"",
"isolation = \"sysbox\"\ncpu_overcommit = 1.0",
);
engine.reload(Config::parse(&with_budget).unwrap(), None, Some(2000));
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
engine.publish();
assert_eq!(
handle.status.borrow().budget.cpu,
Some(api::CpuBudget {
cpus: 2000,
budget: 2000,
in_use: 2000,
held: true,
})
);
}
fn runners_of(
engine: &mut Engine<FakeGithub, FakeDocker>,
handle: &Handle,
repository: &str,
) -> usize {
engine.publish();
handle
.status
.borrow()
.profiles
.iter()
.flat_map(|profile| &profile.slots)
.filter(|slot| {
slot.runner
.as_ref()
.is_some_and(|runner| runner.repository == repository)
})
.count()
}
fn lint(id: u64) -> QueuedJob {
QueuedJob {
name: "lint".to_owned(),
labels: vec!["self-hosted".to_owned(), "light".to_owned()],
..job(id)
}
}
#[tokio::test]
async fn a_heavy_job_keeps_its_place_for_cpu_through_high_water_ticks_and_starts_once_the_cpu_drains()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = format!(
"{}\n[[repositories]]\nname = \"o/b\"\nprofile = \"app\"\nlabels = [\"self-hosted\", \"light\"]\nwarm = 0\n",
cpu_measured()
.replace("cpus = 2", "cpus = 4")
.replace("slots = 6", "slots = 8")
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(10 * G);
engine.cpus = Some(4000);
history(&handle.store, G, 0).await;
history_of(&handle.store, "lint", 2000, G, 0).await;
let db = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
for statement in [
"UPDATE jobs SET cpu_average = 4000 WHERE name = 'build'",
"UPDATE jobs SET cpu_average = 1000, repository = 'o/b' WHERE name = 'lint'",
"UPDATE runners SET repository = 'o/b' WHERE name LIKE 'earlier-2%'",
] {
db.execute(statement, []).unwrap();
}
engine.recover().await.unwrap();
slice_at(&handle.store, 2 * G);
github.0.lock().unwrap().queued = (10..13).map(lint).collect();
engine.tick(Shutdown::No).await;
assert_eq!(runners_of(&mut engine, &handle, "o/b"), 3);
{
let mut state = github.0.lock().unwrap();
state.queued.clear();
for runner in &mut state.runners {
runner.busy = true;
}
}
github.0.lock().unwrap().queued = vec![job(20)];
engine.tick(Shutdown::No).await;
engine.publish();
let budget = handle.status.borrow().budget.clone();
assert!(
budget.cpu.is_some_and(|cpu| cpu.held),
"the build waits for CPU: {budget:?}"
);
engine
.repositories
.listed_mut("o/r")
.and_then(|state| state.waits_mut("app"))
.and_then(|waits| waits.cpu_mut())
.expect("the build's wait is kept")
.since -= PRIORITY_AFTER;
let mut next = 30;
for round in 0..6 {
slice_at(&handle.store, 95 * G / 10);
github.0.lock().unwrap().queued.push(lint(next));
next += 1;
engine.tick(Shutdown::No).await;
slice_at(&handle.store, 2 * G);
github.0.lock().unwrap().queued.push(lint(next));
next += 1;
if round % 2 == 1 {
let mut state = docker.0.lock().unwrap();
if let Some(at) = state.containers.iter().position(|container| {
container
.labels
.get("runnerd.repository")
.map(String::as_str)
== Some("o/b")
}) {
state.containers.remove(at);
}
}
engine.tick(Shutdown::No).await;
if runners_of(&mut engine, &handle, "o/r") == 1 {
break;
}
assert!(
runners_of(&mut engine, &handle, "o/b") <= 3,
"no younger lint takes the CPU the build waits for (round {round})"
);
}
assert_eq!(
runners_of(&mut engine, &handle, "o/r"),
1,
"the build started once the lints it waited for drained"
);
assert_eq!(runners_of(&mut engine, &handle, "o/b"), 0);
assert_eq!(
engine.waits_of("o/r", "app").unwrap(),
crate::scheduler::Waits::default().with_served(true),
"started, the build no longer waits, and the barrier served its pool"
);
}
#[tokio::test]
async fn a_job_started_beside_another_jobs_cpu_wait_does_not_say_it_waited_for_cpu() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = measured().replace(
"admission = \"measured\"",
"admission = \"measured\"\ncpu_overcommit = 1.0",
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(10 * G);
engine.cpus = Some(2000);
history(&handle.store, G, 0).await;
rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.execute("UPDATE jobs SET cpu_average = 1500", [])
.unwrap();
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
slice_at(&handle.store, 2 * G);
engine.set_cpu_wait(
"o/r",
"app",
crate::scheduler::Wait {
job: 2,
since: std::time::SystemTime::now() - Duration::from_mins(5),
},
);
engine.tick(Shutdown::No).await;
let runners = registered(&github.calls())
.iter()
.map(|call| call.trim_start_matches("register ").to_owned())
.collect::<Vec<_>>();
assert_eq!(runners.len(), 1, "job 2 still lacks CPU");
handle.store.claimed(1, 1, &runners[0], time::now_ms());
handle.store.flush().await;
let waited = |admission: Option<api::Admission>| {
admission.and_then(|a| a.cpu).and_then(|cpu| cpu.waited)
};
let admission = handle.store.job(1).await.unwrap().unwrap().admission;
assert_eq!(waited(admission), None, "job 1 never waited for CPU");
engine.cpus = Some(4000);
engine.tick(Shutdown::No).await;
let runners = registered(&github.calls())
.iter()
.map(|call| call.trim_start_matches("register ").to_owned())
.collect::<Vec<_>>();
assert_eq!(runners.len(), 2);
handle.store.claimed(2, 1, &runners[1], time::now_ms());
handle.store.flush().await;
let admission = handle.store.job(2).await.unwrap().unwrap().admission;
assert!(
waited(admission).is_some(),
"job 2 waited for CPU, and its record says so"
);
}
#[tokio::test]
async fn a_job_held_for_cpu_that_leaves_the_queue_no_longer_waits() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = CONFIG.replace(
"isolation = \"sysbox\"",
"isolation = \"sysbox\"\ncpu_overcommit = 1.0",
);
let (mut engine, _handle) = engine_with(&github, &docker, &dir, &config);
engine.cpus = Some(2000);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(
engine
.waits_of("o/r", "app")
.and_then(|waits| waits.cpu())
.map(|wait| wait.job),
Some(2)
);
github.0.lock().unwrap().queued = vec![job(1)];
engine.tick(Shutdown::No).await;
assert!(
engine.keeps_no_waits("o/r"),
"a job gone from the queue no longer waits"
);
}
pub(super) fn host_at(store: &Store, slice: u64, available: u64, stalled: u32) {
store.host_memory(api::HostMemory {
at: time::now_ms() / 1000,
total: 64 * G,
available,
slice,
some: Some(api::Pressure::default()),
full: Some(api::Pressure {
avg10: stalled,
avg60: 0,
}),
holders: vec![
api::Holder {
name: "java".to_owned(),
memory: 9 * G,
},
api::Holder {
name: "qemu-system-x86".to_owned(),
memory: 6 * G,
},
],
ceiling: None,
swap: None,
});
}
fn waited(dir: &TestDir, column: &str) -> Vec<u64> {
let db = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
let mut statement = db
.prepare(&format!(
"SELECT {column} FROM jobs WHERE id <= 4 ORDER BY id"
))
.unwrap();
statement
.query_map([], |row| row.get(0))
.unwrap()
.collect::<Result<_, _>>()
.unwrap()
}
fn waits(handle: &Handle) -> Vec<Option<api::Waiting>> {
handle
.status
.borrow()
.queued
.iter()
.map(|job| job.waiting.clone())
.collect()
}
#[tokio::test]
async fn a_planned_start_that_stops_a_warm_runner_leaves_the_plain_margin() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 0);
host_at(&handle.store, 0, 5 * G, 0);
let (_, admitting) = engine.plan();
let slot = |index| SlotRef {
profile: "app".to_owned(),
index,
};
let replacing = Plan {
decisions: vec![Decision::Start {
slot: slot(1),
repository: "o/r".to_owned(),
memory: 2 * G,
cpu: 2000,
for_: StartFor::Job(1),
replacing: vec![slot(0)],
behind: None,
}],
..Plan::default()
};
engine.note_host(&admitting, &replacing);
assert!(
!engine.host_is_short(),
"a start that stops a warm runner keeps the plain margin"
);
let waiting = Plan {
waiting: BTreeMap::from([(
1,
api::Waiting::HostMemory {
needs: 2 * G,
free: 0,
},
)]),
..Plan::default()
};
engine.note_host(&admitting, &waiting);
assert!(
engine.host_is_short(),
"a 2G waiter raises the larger margin"
);
}
#[tokio::test]
async fn jobs_wait_while_the_host_is_short_of_memory_and_start_once_it_has_room_again() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 0);
host_at(&handle.store, 0, 4 * G + 2 * 1280 * M + 100 * M, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
engine.publish();
let short = Some(api::Waiting::HostMemory {
needs: 1280 * M,
free: 0,
});
assert_eq!(
waits(&handle)[2..],
[short.clone(), short],
"the two left wait for the host's memory"
);
handle.store.flush().await;
let hold = handle.store.insights(0).await.unwrap().host.unwrap();
assert_eq!(hold.jobs.len(), 2, "the two left wait: {hold:?}");
assert_eq!(hold.memory.holders[0].name, "java");
let reason = {
let status = handle.status.borrow();
let job = &status.queued[2];
job.waiting.as_ref().unwrap().reason(job, &status, 0)
};
assert!(
reason.starts_with("The host is short of memory"),
"{reason}"
);
let outside = crate::config::ByteSize(64 * G - (4 * G + 2 * 1280 * M + 100 * M));
assert!(
reason.ends_with(&format!(
"programs outside CI hold {outside}, most java 9G and qemu-system-x86 6G"
)),
"{reason}"
);
tokio::time::sleep(Duration::from_millis(20)).await;
host_at(&handle.store, 0, 4 * G + 3 * 1280 * M + 100 * M, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "the larger margin");
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
3,
"one fits the plain one"
);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 3, "no room is left");
assert!(engine.host_is_short(), "no waiter fits the plain margin");
tokio::time::sleep(Duration::from_millis(20)).await;
host_at(&handle.store, 0, 6 * G + 4 * 1280 * M + 100 * M, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4);
handle.store.flush().await;
assert_eq!(
handle.store.insights(0).await.unwrap().host,
None,
"once they start, no job waits for the host's memory"
);
let waited = waited(&dir, "host_waited");
assert_eq!(waited[..2], [0, 0], "the jobs that started waited not");
assert!(
waited[2] >= 20 && waited[3] >= 40,
"each that waited counts the ticks it waited: {waited:?}"
);
assert!(removed(&docker.calls()).is_empty(), "no runner was stopped");
}
#[tokio::test]
async fn jobs_count_a_wait_for_a_slot_only_while_memory_had_room_for_their_runners() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(
&github,
&docker,
&dir,
&measured().replace("slots = 4", "slots = 2"),
);
engine.memory_max = Some(3 * 1280 * M + 100 * M);
history(&handle.store, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
tokio::time::sleep(Duration::from_millis(20)).await;
engine.tick(Shutdown::No).await;
handle.store.flush().await;
let waited = waited(&dir, "slot_waited");
assert_eq!(
waited.iter().map(|&ms| ms >= 20).collect::<Vec<_>>(),
[false, false, true, false],
"only job 3 waited for a slot alone: {waited:?}"
);
}
#[tokio::test]
async fn a_job_a_pass_starts_stops_waiting_for_a_slot_when_its_runner_starts() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(
&github,
&docker,
&dir,
&measured().replace("slots = 4", "slots = 2"),
);
history(&handle.store, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=3).map(job).collect();
slice_at(&handle.store, 0);
let began = std::time::Instant::now();
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
docker.0.lock().unwrap().containers[0].running = false;
engine.exits.any();
engine.turnover().await;
assert_eq!(
registered(&github.calls()).len(),
3,
"the pass starts job 3's runner"
);
let until_pass_ended = u64::try_from(began.elapsed().as_millis()).unwrap() + 1;
tokio::time::sleep(Duration::from_millis(300)).await;
engine.tick(Shutdown::No).await;
handle.store.flush().await;
let waited = waited(&dir, "slot_waited");
assert!(
waited.iter().all(|&ms| ms <= until_pass_ended),
"no job waits for a slot past its runner's start, {until_pass_ended} ms in: {waited:?}"
);
}
async fn third_job_waits_for_a_slot(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let (mut engine, handle) = engine_with(
github,
docker,
dir,
&measured().replace("slots = 4", "slots = 2"),
);
history(&handle.store, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=3).map(job).collect();
slice_at(&handle.store, 0);
engine.tick(Shutdown::No).await;
assert!(engine.waits_for_a_slot(3), "job 3 waits for a slot");
(engine, handle)
}
#[tokio::test]
async fn the_job_a_runner_took_stops_waiting_when_its_capture_names_it() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = third_job_waits_for_a_slot(&github, &docker, &dir).await;
tokio::time::sleep(Duration::from_millis(20)).await;
let runner = engine.occupied().next().unwrap().1.runner_name.clone();
engine.note_found(&crate::scheduler::engine::history::Found::named(runner, 3));
assert!(!engine.waits_for_a_slot(3), "job 3 no longer waits");
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
tokio::time::sleep(Duration::from_millis(300)).await;
engine.tick(Shutdown::No).await;
handle.store.flush().await;
let third = waited(&dir, "slot_waited")[2];
assert!(
(20..200).contains(&third),
"job 3 waited until it was named: {third} ms"
);
}
#[tokio::test]
async fn a_job_waits_for_a_slot_or_the_hosts_memory_never_both() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, _handle) = third_job_waits_for_a_slot(&github, &docker, &dir).await;
engine.note_host_waits(BTreeSet::from([3]), time::now_ms());
assert!(
!engine.waits_for_a_slot(3),
"a job now short of the host's memory left the slot wait"
);
engine.note_slots(BTreeSet::from([3]));
assert!(
!engine.waits_for_the_host(3),
"a job now short of a slot left the host wait"
);
}
#[tokio::test]
async fn a_pass_keeps_only_the_waits_of_the_starts_it_did_not_make() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, _handle) = engine_with(&github, &docker, &dir, &measured());
engine.note_slots(BTreeSet::from([1, 2]));
engine.note_host_waits(BTreeSet::from([3, 4, 6]), time::now_ms());
let needs_memory = api::Waiting::HostMemory {
needs: 1280 * M,
free: 0,
};
let plan = Plan {
slot_waiting: BTreeSet::from([5, 6]),
waiting: BTreeMap::from([(5, needs_memory)]),
..Default::default()
};
engine.note_pass_waits(&plan, &BTreeSet::from([1, 3, 6]));
assert!(engine.waits_for_a_slot(1), "job 1's start was not made");
assert!(!engine.waits_for_a_slot(2), "the plan no longer has job 2");
assert!(engine.waits_for_the_host(3), "job 3's start was not made");
assert!(
!engine.waits_for_the_host(4),
"the plan no longer has job 4"
);
assert!(
engine.waits_for_the_host(5) && !engine.waits_for_a_slot(5),
"a job the plan holds for both ends in the host's wait"
);
assert!(
engine.waits_for_the_host(6) && !engine.waits_for_a_slot(6),
"job 6's start was not made, so it stays in the wait it was in"
);
}
#[tokio::test]
async fn the_first_runner_into_an_empty_slice_needs_only_the_margin_after_a_wait() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 0);
host_at(&handle.store, 0, 4 * G + 1000 * M, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 0, "1.25G does not fit");
host_at(&handle.store, 0, 4 * G + 1280 * M + 100 * M, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
}
#[tokio::test]
async fn a_runner_using_more_than_it_reserved_takes_what_it_uses_from_the_host() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
github.0.lock().unwrap().queued = vec![job(1)];
slice_at(&handle.store, 0);
host_at(&handle.store, 0, 40 * G, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
runners_use(&github, &handle.store, 3 * G);
slice_at(&handle.store, 3 * G);
host_at(&handle.store, 3 * G, 4 * G + 1280 * M + 100 * M, 0);
github.0.lock().unwrap().queued = (1..=3).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
}
#[tokio::test]
async fn under_caps_a_runner_starts_only_if_its_cap_fits_what_the_host_leaves() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, CONFIG);
engine.memory_max = Some(20 * G);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
host_at(&handle.store, 0, 8 * G + 100 * M, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
engine.publish();
assert_eq!(
waits(&handle)[1],
Some(api::Waiting::HostMemory {
needs: 4 * G,
free: 0
})
);
}
fn idle_vm_at(store: &Store, stalled: u32) {
store.host_memory(api::HostMemory {
at: time::now_ms() / 1000,
total: 2 * G + 120 * M,
available: G + 490 * M,
slice: 0,
some: Some(api::Pressure::default()),
full: Some(api::Pressure {
avg10: stalled,
avg60: 0,
}),
holders: Vec::new(),
ceiling: None,
swap: None,
});
}
async fn vm_engine(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
host: &str,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let config = CONFIG.replace(
"isolation = \"sysbox\"",
&format!("isolation = \"sysbox\"\n{host}"),
);
let (mut engine, handle) = engine_with(github, docker, dir, &config);
engine.memory_max = Some(20 * G);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
(engine, handle)
}
#[tokio::test]
async fn an_idle_vm_without_a_memory_ceiling_starts_nothing() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = vm_engine(&github, &docker, &dir, "").await;
idle_vm_at(&handle.store, 0);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
0,
"1.48G is under the 2G margin"
);
engine.publish();
assert_eq!(
waits(&handle)[0],
Some(api::Waiting::HostMemory {
needs: 4 * G,
free: 0
})
);
}
#[tokio::test]
async fn a_vm_that_can_grow_to_its_memory_ceiling_starts_runners_in_what_it_can_grow_by() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = vm_engine(&github, &docker, &dir, "memory_ceiling = \"24G\"").await;
idle_vm_at(&handle.store, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
engine.publish();
let line = handle.status.borrow().budget.host_line().unwrap();
assert!(
line.starts_with("host: 1.48G of 2.12G available, growing to 24G; leaves the slice 21.36G"),
"{line}"
);
}
#[tokio::test]
async fn a_vm_that_stalls_short_of_memory_starts_nothing_whatever_its_memory_ceiling() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = vm_engine(&github, &docker, &dir, "memory_ceiling = \"24G\"").await;
idle_vm_at(&handle.store, 600);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 0, "nothing starts");
engine.publish();
let status = handle.status.borrow();
assert!(
status.budget.host.as_ref().is_some_and(|host| host.stalled),
"the budget says the host stalls"
);
let job = &status.queued[0];
let reason = job.waiting.as_ref().unwrap().reason(job, &status, 0);
assert!(reason.contains("stall"), "{reason}");
}
#[tokio::test]
async fn a_runner_whose_room_the_host_lost_since_the_plan_does_not_start_in_the_same_tick() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, CONFIG);
engine.memory_max = Some(20 * G);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
host_at(&handle.store, 0, 20 * G, 0);
let store = handle.store.clone();
docker.0.lock().unwrap().on_prepare = Some(Box::new(move |_| {
host_at(&store, 0, 8 * G + 100 * M, 0);
}));
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
1,
"the second start waits for room"
);
engine.publish();
assert_eq!(
waits(&handle)[1],
Some(api::Waiting::HostMemory {
needs: 4 * G,
free: 0
})
);
}
#[tokio::test]
async fn nothing_starts_while_the_host_stalls_short_of_memory() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 0);
host_at(&handle.store, 10 * G, 7 * G + 512 * M, 600);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 0, "nothing starts");
engine.publish();
{
let status = handle.status.borrow();
assert!(
status.budget.host.as_ref().is_some_and(|host| host.stalled),
"the budget says the host stalls"
);
let job = status.queued.last().unwrap();
let reason = job.waiting.as_ref().unwrap().reason(job, &status, 0);
assert!(reason.contains("stall"), "{reason}");
let outside = crate::config::ByteSize(64 * G - 7 * G - 512 * M - 10 * G);
assert!(
reason.ends_with(&format!(
"programs outside CI hold {outside}, most java 9G and qemu-system-x86 6G"
)),
"{reason}"
);
}
host_at(&handle.store, 0, 7 * G + 512 * M, 300);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 0, "still held at 3%");
host_at(&handle.store, 0, 7 * G + 512 * M, 50);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "resumed under 1%");
}
#[tokio::test]
async fn a_stall_with_memory_to_spare_holds_no_start_and_stops_no_runner() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 0);
github.0.lock().unwrap().queued = vec![job(1)];
host_at(&handle.store, 0, 40 * G, 0);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1);
github.0.lock().unwrap().queued = (1..=3).map(job).collect();
host_at(&handle.store, 0, 40 * G, 600);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
3,
"the stall holds nothing"
);
engine.publish();
assert!(
handle
.status
.borrow()
.budget
.host
.as_ref()
.is_some_and(|host| !host.stalled),
"the budget says the host does not stall"
);
assert!(removed(&docker.calls()).is_empty(), "no runner was stopped");
}
#[tokio::test]
async fn without_a_recent_reading_of_the_host_only_the_slice_bounds_starts() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 0);
handle.store.host_memory(api::HostMemory {
at: time::now_ms() / 1000 - 60,
total: 64 * G,
available: 0,
slice: 0,
some: None,
full: None,
holders: Vec::new(),
ceiling: None,
swap: None,
});
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4);
}
#[tokio::test]
async fn runners_a_restart_adopts_keep_the_cpu_they_reserved_not_their_limits() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &cpu_measured());
engine.cpus = Some(4000);
history(&handle.store, G, 0).await;
rusqlite::Connection::open(dir.0.join("history.db"))
.unwrap()
.execute("UPDATE jobs SET cpu_average = 500", [])
.unwrap();
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4, "4 × 0.5 CPU fit 4");
handle.store.flush().await;
drop((engine, handle));
let (mut engine, handle) = engine_with(&github, &docker, &dir, &cpu_measured());
engine.cpus = Some(4000);
engine.recover().await.unwrap();
engine.publish();
assert_eq!(
handle.status.borrow().budget.cpu.map(|cpu| cpu.in_use),
Some(2000)
);
}
#[tokio::test]
async fn a_start_that_no_longer_fits_does_not_hold_back_the_plans_later_starts() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = sized("0G", "4G", "512M").replacen("slots = 1", "slots = 2", 1);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(20 * G);
engine.recover().await.unwrap();
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(1), job(2), small_job(3)];
state
.queued_on
.extend([(1, "o/r".to_owned()), (2, "o/r".to_owned())]);
state.queued_on.insert(3, "o/s".to_owned());
}
host_at(&handle.store, 0, 20 * G, 0);
let store = handle.store.clone();
docker.0.lock().unwrap().on_prepare = Some(Box::new(move |_| {
host_at(&store, 0, 9 * G + 100 * M, 0);
}));
engine.tick(Shutdown::No).await;
let registered: Vec<String> = registered(&github.calls()).into_iter().cloned().collect();
assert_eq!(registered.len(), 2, "{registered:?}");
assert!(
registered[1].contains("-small-"),
"o/s's runner starts in the same tick: {registered:?}"
);
engine.publish();
assert_eq!(
waits(&handle)[1],
Some(api::Waiting::HostMemory {
needs: 4 * G,
free: 0
})
);
}
#[tokio::test]
async fn a_stop_asked_for_while_a_runner_registers_registers_no_more_of_the_tick() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = CONFIG.replace("slots = 2", "slots = 4");
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
let control = handle.control.clone();
github.0.lock().unwrap().on_register = Some(Box::new(move |_| {
control.shutdown(Shutdown::Detach);
}));
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 1, "{:?}", github.calls());
assert_eq!(
github.0.lock().unwrap().runners,
Vec::new(),
"its registration is deleted"
);
let started = docker
.calls()
.iter()
.filter(|call| call.starts_with("start "))
.count();
assert_eq!(started, 0, "{:?}", docker.calls());
}
#[tokio::test]
async fn a_pass_raises_what_waiting_runners_reserve_before_it_admits() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
github.0.lock().unwrap().queued = vec![job(1), job(2)];
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "1.25G for each build");
let ended = {
let occupant = engine.occupants_mut().last().unwrap();
occupant.has_taken(2);
occupant.container.clone()
};
for container in &mut docker.0.lock().unwrap().containers {
if container.id == ended {
container.running = false;
}
}
let e2e = |id| QueuedJob {
name: "e2e".to_owned(),
..job(id)
};
engine.set_queued("o/r", vec![job(1), e2e(5), e2e(6)]);
engine.exits.stopped(ended);
engine.turnover().await;
engine.rereserve().await;
engine.publish();
let in_use = handle.status.borrow().budget.memory_in_use;
assert!(
in_use <= 10 * G,
"the runners reserve {in_use} bytes of the 10G slice"
);
}
#[tokio::test]
async fn waiting_runners_a_pass_raised_past_the_budget_are_stopped_by_the_next_tick() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4, "4 × 1.25G");
let ended = {
let occupant = engine.occupants_mut().last().unwrap();
occupant.has_taken(4);
occupant.container.clone()
};
for container in &mut docker.0.lock().unwrap().containers {
if container.id == ended {
container.running = false;
}
}
let e2e = QueuedJob {
name: "e2e".to_owned(),
..job(5)
};
let queued = vec![job(1), job(2), job(3), e2e];
engine.set_queued("o/r", queued.clone());
github.0.lock().unwrap().queued = queued;
engine.exits.stopped(ended);
engine.turnover().await;
engine.publish();
assert_eq!(
handle.status.borrow().budget.memory_in_use,
3 * 3584 * M,
"the pass raised the three left"
);
engine.tick(Shutdown::No).await;
engine.publish();
let in_use = handle.status.borrow().budget.memory_in_use;
assert!(
in_use <= 10 * G,
"the runners reserve {in_use} bytes of the 10G slice"
);
}
fn unpooled() -> String {
measured().replace(
"admission = \"measured\"",
"admission = \"measured\"\npooled_count = false",
)
}
pub(super) fn roomy_readings(github: &FakeGithub, store: &Store, used: u64) {
runners_use(github, store, used);
slice_at(store, G);
host_at(store, G, 40 * G, 0);
}
pub(super) fn first_minute_passes(engine: &mut Engine<FakeGithub, FakeDocker>) {
for occupant in engine.occupants_mut() {
occupant.age_job_start(POOL_AFTER);
}
}
pub(super) async fn two_settled_jobs(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
config: &str,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let (mut engine, handle) = engine_with(github, docker, dir, config);
engine.memory_max = Some(8 * G);
history_of(&handle.store, "build", 1000, 3 * G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
history_of(&handle.store, "lint", 3000, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![
job(1),
QueuedJob {
name: "e2e".to_owned(),
..job(2)
},
];
for slot in 1..=2 {
let capture = format!("@@job {slot} {slot}\n");
docker.0.lock().unwrap().slot_captures.insert(slot, capture);
}
roomy_readings(github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
until_state(&handle, 1, JobState::Running).await;
until_state(&handle, 2, JobState::Running).await;
github.0.lock().unwrap().queued = Vec::new();
roomy_readings(github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(github, &handle.store, 512 * M);
(engine, handle)
}
fn small(id: u64) -> QueuedJob {
QueuedJob {
name: "lint".to_owned(),
..job(id)
}
}
fn small_host_at(store: &Store, available: u64) {
store.host_memory(api::HostMemory {
at: time::now_ms() / 1000,
total: 16 * G,
available,
slice: G,
some: Some(api::Pressure::default()),
full: Some(api::Pressure::default()),
holders: Vec::new(),
ceiling: None,
swap: None,
});
}
fn only_reason(handle: &Handle) -> String {
let status = handle.status.borrow();
let job = &status.queued[0];
job.waiting.as_ref().unwrap().reason(job, &status, 0)
}
async fn host_held_insights(engine: &Engine<FakeGithub, FakeDocker>, handle: &Handle) -> usize {
handle.store.flush().await;
let history = handle.store.insights(0).await.unwrap();
crate::insights::insights(&history, &engine.config, 7, time::now_ms())
.insights
.iter()
.filter(|insight| matches!(insight.kind, api::InsightKind::HostHeld { .. }))
.count()
}
#[tokio::test]
async fn a_job_the_host_limit_holds_with_room_free_asks_nobody_to_close_programs() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
github.0.lock().unwrap().queued = vec![small(3)];
runners_use(&github, &handle.store, 512 * M);
slice_at(&handle.store, G);
small_host_at(&handle.store, 8 * G);
engine.tick(Shutdown::No).await;
tokio::time::sleep(Duration::from_millis(20)).await;
runners_use(&github, &handle.store, 512 * M);
slice_at(&handle.store, G);
small_host_at(&handle.store, 8 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert!(
handle.status.borrow().budget.pooled.is_some(),
"a credit opens the slice"
);
assert_eq!(
waits(&handle),
[Some(api::Waiting::HostMemory {
needs: 1280 * M,
free: 0
})],
"the host limit reads the plain count, so it holds the job"
);
assert_eq!(
only_reason(&handle),
"The host has 8G of 16G available, but what runs may still grow into it: needs 1.25G; 0B of the 6G it leaves \
the slice is free"
);
handle.store.flush().await;
assert!(
waited(&dir, "host_waited")[2] >= 20,
"the job waited for the host's limit"
);
handle.store.host_waited(vec![3], 60 * 60_000);
assert_eq!(
host_held_insights(&engine, &handle).await,
0,
"6G of room past the margin, and the job needs 1.25G"
);
engine.memory_max = Some(9 * G);
runners_use(&github, &handle.store, 512 * M);
slice_at(&handle.store, G);
small_host_at(&handle.store, 3 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert!(
only_reason(&handle).starts_with("The host is short of memory: needs 1.25G"),
"{}",
only_reason(&handle)
);
assert_eq!(
host_held_insights(&engine, &handle).await,
1,
"1G past the margin, and the job needs 1.25G"
);
runners_use(&github, &handle.store, 512 * M);
slice_at(&handle.store, G);
small_host_at(&handle.store, 3 * G + 512 * M);
engine.tick(Shutdown::No).await;
engine.publish();
assert!(
only_reason(&handle).starts_with("The host is short of memory: needs 1.25G"),
"{}",
only_reason(&handle)
);
assert_eq!(
host_held_insights(&engine, &handle).await,
1,
"1.5G past the margin, under one more margin of 2G"
);
runners_use(&github, &handle.store, 512 * M);
slice_at(&handle.store, G);
small_host_at(&handle.store, 8 * G);
engine.tick(Shutdown::No).await;
assert_eq!(host_held_insights(&engine, &handle).await, 0);
}
#[tokio::test]
async fn settled_jobs_of_two_kinds_make_room_a_start_goes_by_and_every_reader_agrees() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
engine.publish();
let budget = handle.status.borrow().budget.clone();
let pooled = budget.pooled.expect("two settled jobs, room above use");
assert_eq!(budget.memory_in_use, 7 * G, "the plain count stays a fact");
assert_eq!(pooled.counted, 7 * G - pooled.credit);
assert_eq!(pooled.runners, 2);
assert_eq!(pooled.credit / M, 1799);
let remembered = engine.host_remembered();
assert_eq!(engine.count(remembered).pooled(), pooled.counted);
let plan = engine.plan_budget(&engine.admitting(&mut [], None));
assert_eq!(
plan.memory,
Some(crate::scheduler::left(8 * G, pooled.counted)),
"the plan starts from what the status shows"
);
assert_eq!(
budget.host.map(|host| host.spare),
Some(G + 40 * G - 4 * G - budget.memory_in_use),
"two jobs keep all their headroom for growth: the host's limit reads the plain count"
);
assert_eq!(
plan.host,
Some(crate::scheduler::left(
G + 40 * G - 4 * G,
budget.memory_in_use
)),
"and so does a plan's"
);
github.0.lock().unwrap().queued = vec![small(3)];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 3, "lint starts");
engine.publish();
let budget = handle.status.borrow().budget.clone();
assert_eq!(budget.memory_in_use, 8 * G + 256 * M, "past 8G, in full");
assert_eq!(
budget.pooled.map(|after| after.credit),
Some(pooled.credit),
"a runner just started changes no credit"
);
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
let stopped = github
.calls()
.iter()
.filter(|call| call.starts_with("delete "))
.count();
assert_eq!(stopped, 0, "what fits the pooled count stays");
}
#[tokio::test]
async fn a_raised_waiting_runner_stops_only_once_the_pooled_count_is_past_the_budget() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
github.0.lock().unwrap().queued = vec![small(3)];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 3, "lint's runner waits");
github.0.lock().unwrap().queued = Vec::new();
let waiting = engine
.occupied()
.find(|(_, occupant)| occupant.named_job().is_none())
.map(|(_, occupant)| occupant.runner_name.clone())
.unwrap();
engine.raise(waiting);
roomy_readings(&github, &handle.store, 512 * M);
assert_eq!(engine.over_budget(&[]), [], "the pooled count fits");
handle.store.host_memory(api::HostMemory {
at: time::now_ms() / 1000 - 60,
total: 64 * G,
available: 40 * G,
slice: G,
some: None,
full: None,
holders: Vec::new(),
ceiling: None,
swap: None,
});
assert_eq!(engine.over_budget(&[]).len(), 1, "the plain count does not");
}
#[tokio::test]
async fn a_reload_back_to_caps_stops_no_runner_measured_admission_raised() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
github.0.lock().unwrap().queued = vec![small(3)];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 3, "lint's runner waits");
github.0.lock().unwrap().queued = Vec::new();
let waiting = engine
.occupied()
.find(|(_, occupant)| occupant.named_job().is_none())
.map(|(_, occupant)| occupant.runner_name.clone())
.unwrap();
engine.raise(waiting);
let caps = CONFIG.replace("slots = 2", "slots = 4");
engine.reload(Config::parse(&caps).unwrap(), Some(8 * G), None);
engine.rereserve().await;
roomy_readings(&github, &handle.store, 512 * M);
handle.store.host_memory(api::HostMemory {
at: time::now_ms() / 1000 - 60,
total: 64 * G,
available: 40 * G,
slice: G,
some: None,
full: None,
holders: Vec::new(),
ceiling: None,
swap: None,
});
assert_eq!(
engine.over_budget(&[]),
[],
"no runner is raised under caps"
);
}
#[tokio::test]
async fn the_last_check_before_a_start_keeps_the_margin_its_plan_chose() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
host_at(&handle.store, G, 21 * G, 0);
engine.set_host_short();
let lint = 1280 * M;
let mut plan = engine.admitting(&mut [], None);
access::set_plan_short(&mut plan, false);
assert!(
engine.still_fits(&plan, "app", lint, 0),
"1G left and 1G of credit, by the plan's margin"
);
access::set_plan_short(&mut plan, true);
assert!(
!engine.still_fits(&plan, "app", lint, 0),
"no credit by the larger margin"
);
}
#[tokio::test]
async fn a_reseed_reads_the_room_a_plan_reads() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let second_profile = |config: String| {
config.replace(
"[[repositories]]",
"[profiles.spare]\nimage_stage = \"spare\"\ncpus = 2\nmemory = \"2G\"\nslots = 1\n\n\
[[repositories]]\nname = \"o/s\"\nprofile = \"spare\"\nlabels = [\"spare\"]\nwarm = 0\n\n\
[[repositories]]",
)
};
let (engine, _) = two_settled_jobs(&github, &docker, &dir, &second_profile(measured())).await;
assert!(
engine.fits_seed("spare"),
"2G of the 2.76G the count leaves"
);
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (engine, _) = two_settled_jobs(&github, &docker, &dir, &second_profile(unpooled())).await;
assert!(!engine.fits_seed("spare"), "1G left of the plain count");
}
#[tokio::test]
async fn with_the_pooled_count_off_the_same_jobs_count_and_plan_as_they_add_up() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &unpooled()).await;
engine.publish();
let budget = handle.status.borrow().budget.clone();
assert_eq!(budget.pooled, None);
assert_eq!(budget.memory_in_use, 7 * G);
let plan = engine.plan_budget(&engine.admitting(&mut [], None));
assert_eq!(plan.memory, Some(crate::scheduler::left(8 * G, 7 * G)));
assert_eq!(plan.credit_room, 0, "nothing for a barrier to expect");
github.0.lock().unwrap().queued = vec![small(3)];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "lint's 1.25G waits");
}
#[tokio::test]
async fn caps_admission_pools_nothing() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
engine.config.host.admission = crate::config::Admission::Caps;
engine.publish();
assert_eq!(handle.status.borrow().budget.pooled, None);
assert_eq!(handle.status.borrow().budget.memory_in_use, 8 * G);
}
#[tokio::test]
async fn any_stale_reading_takes_the_credit_back() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
let old = time::now_ms() - 60_000;
let credit = |engine: &mut Engine<FakeGithub, FakeDocker>| {
engine.publish();
let budget = handle.status.borrow().budget.clone();
budget.pooled.map(|pooled| pooled.credit)
};
assert!(credit(&mut engine).is_some(), "the readings are fresh");
handle.store.host_memory(api::HostMemory {
at: old / 1000,
total: 64 * G,
available: 40 * G,
slice: G,
some: Some(api::Pressure::default()),
full: Some(api::Pressure::default()),
holders: Vec::new(),
ceiling: None,
swap: None,
});
assert_eq!(credit(&mut engine), None, "without a recent host reading");
host_at(&handle.store, G, 40 * G, 0);
handle.store.host_sample(api::HostSample {
at: old / 1000,
cpu: 0,
memory: G,
});
assert_eq!(credit(&mut engine), None, "without a recent slice sample");
assert_eq!(handle.status.borrow().budget.memory_in_use, 8 * G);
slice_at(&handle.store, G);
let stale = engine
.occupied()
.map(|(_, occupant)| occupant.runner_name.clone())
.next()
.unwrap();
runners_use(&github, &handle.store, 512 * M);
handle.store.samples(vec![SampleRecord {
runner: stale.clone(),
at: old,
cpu: 0,
cpu_usage: 0,
memory: 512 * M,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
written: None,
dirty: None,
events: Events::default(),
}]);
assert_eq!(credit(&mut engine), None, "its one job left to pool");
}
#[tokio::test]
async fn a_runner_a_restart_adopts_counts_in_full_for_the_rest_of_its_job() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
handle.store.flush().await;
drop((engine, handle));
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
engine.recover().await.unwrap();
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(&github, &handle.store, 512 * M);
engine.publish();
let budget = handle.status.borrow().budget.clone();
assert_eq!(budget.memory_in_use, 7 * G, "both keep what they reserved");
assert_eq!(budget.pooled, None, "this daemon did not see them start");
}
#[tokio::test]
async fn a_runner_a_restart_adopts_that_a_listing_shows_waiting_counts_its_next_job_as_started_here()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
history_of(&handle.store, "build", 1000, 3 * G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![
job(1),
QueuedJob {
name: "e2e".to_owned(),
..job(2)
},
];
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
handle.store.flush().await;
until_captures_end(&engine).await;
drop((engine, handle));
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
engine.recover().await.unwrap();
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
2,
"both wait for the jobs"
);
let runners: Vec<String> = engine
.occupied()
.map(|(_, occupant)| occupant.runner_name.clone())
.collect();
for (job, runner) in (1..).zip(runners) {
engine.note_found(&crate::scheduler::engine::history::Found::named(
runner, job,
));
}
{
let mut state = github.0.lock().unwrap();
state.queued = Vec::new();
for runner in &mut state.runners {
runner.busy = true;
}
}
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(&github, &handle.store, 512 * M);
engine.publish();
assert!(
handle.status.borrow().budget.pooled.is_some(),
"both jobs started under this daemon and have settled"
);
}
#[tokio::test]
async fn a_runner_a_restart_adopts_busy_does_not_count_a_job_named_later_as_started_here() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
history_of(&handle.store, "build", 1000, 3 * G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![
job(1),
QueuedJob {
name: "e2e".to_owned(),
..job(2)
},
];
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
handle.store.flush().await;
until_captures_end(&engine).await;
drop((engine, handle));
{
let mut state = github.0.lock().unwrap();
state.queued = Vec::new();
for runner in &mut state.runners {
runner.busy = true;
}
}
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
engine.recover().await.unwrap();
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
let runners: Vec<String> = engine
.occupied()
.map(|(_, occupant)| occupant.runner_name.clone())
.collect();
for (job, runner) in (1..).zip(runners) {
engine.note_found(&crate::scheduler::engine::history::Found::named(
runner, job,
));
}
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(&github, &handle.store, 512 * M);
engine.publish();
assert_eq!(
handle.status.borrow().budget.pooled,
None,
"their runners are not known to have started as predicted"
);
}
#[tokio::test]
async fn a_waiting_runner_a_restart_adopts_counts_in_full_once_it_takes_a_job() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
history_of(&handle.store, "build", 1000, 3 * G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![
job(1),
QueuedJob {
name: "e2e".to_owned(),
..job(2)
},
];
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
handle.store.flush().await;
until_captures_end(&engine).await;
drop((engine, handle));
for slot in 1..=2 {
let capture = format!("@@job {slot} {slot}\n");
docker.0.lock().unwrap().slot_captures.insert(slot, capture);
}
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
until_state(&handle, 1, JobState::Running).await;
until_state(&handle, 2, JobState::Running).await;
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(&github, &handle.store, 512 * M);
let found = engine
.occupied()
.filter(|(_, occupant)| occupant.job_found_at().is_some() && occupant.has_key())
.count();
assert_eq!(found, 2, "this daemon found both jobs");
engine.publish();
assert_eq!(
handle.status.borrow().budget.pooled,
None,
"their runners are not known to have started as predicted"
);
}
#[tokio::test]
async fn a_runner_a_restart_adopts_whose_runners_listing_fails_does_not_count_its_job_as_started_here()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
history_of(&handle.store, "build", 1000, 3 * G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![
job(1),
QueuedJob {
name: "e2e".to_owned(),
..job(2)
},
];
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
handle.store.flush().await;
until_captures_end(&engine).await;
drop((engine, handle));
for slot in 1..=2 {
let capture = format!("@@job {slot} {slot}\n");
docker.0.lock().unwrap().slot_captures.insert(slot, capture);
}
github
.0
.lock()
.unwrap()
.failed_runners
.insert("o/r".to_owned(), reqwest::StatusCode::INTERNAL_SERVER_ERROR);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(8 * G);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
until_state(&handle, 1, JobState::Running).await;
until_state(&handle, 2, JobState::Running).await;
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(&github, &handle.store, 512 * M);
let found = engine
.occupied()
.filter(|(_, occupant)| occupant.job_found_at().is_some() && occupant.has_key())
.count();
assert_eq!(found, 2, "this daemon found both jobs");
engine.publish();
assert_eq!(
handle.status.borrow().budget.pooled,
None,
"their runners are not known to have started as predicted"
);
}
#[tokio::test]
async fn a_host_that_stalls_or_runs_short_takes_the_credit_back_and_holds_the_start() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = two_settled_jobs(&github, &docker, &dir, &measured()).await;
host_at(&handle.store, G, 19 * G, 0);
engine.publish();
assert_eq!(handle.status.borrow().budget.pooled, None);
github.0.lock().unwrap().queued = vec![small(3)];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2, "lint waits again");
}
async fn three_settled_jobs(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
config: &str,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
let (mut engine, handle) = engine_with(github, docker, dir, config);
engine.memory_max = Some(32 * G);
history_of(&handle.store, "build", 1000, 3 * G, 0).await;
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
history_of(&handle.store, "it", 3000, 3 * G, 0).await;
history_of(&handle.store, "lint", 4000, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = ["build", "e2e", "it"]
.into_iter()
.zip(1..)
.map(|(name, id)| QueuedJob {
name: name.to_owned(),
..job(id)
})
.collect();
for slot in 1..=3 {
let capture = format!("@@job {slot} {slot}\n");
docker.0.lock().unwrap().slot_captures.insert(slot, capture);
}
roomy_readings(github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 3);
for id in 1..=3 {
until_state(&handle, id, JobState::Running).await;
}
github.0.lock().unwrap().queued = Vec::new();
roomy_readings(github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
roomy_readings(github, &handle.store, 512 * M);
(engine, handle)
}
fn tight_host(github: &FakeGithub, store: &Store, available: u64) {
runners_use(github, store, 512 * M);
slice_at(store, G);
host_at(store, G, available, 0);
}
#[tokio::test]
async fn a_host_credit_starts_what_only_it_fits_and_a_restart_or_pooled_count_off_holds_it() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = three_settled_jobs(&github, &docker, &dir, &measured()).await;
tight_host(&github, &handle.store, 12 * G);
engine.publish();
let budget = handle.status.borrow().budget.clone();
let host = budget.host.clone().expect("a fresh reading");
assert_eq!(budget.memory_in_use, 10 * G + 512 * M);
assert_eq!(
host.credit,
Some(api::HostCredit {
bytes: 3 * G,
used: G + 512 * M,
})
);
assert_eq!(host.spare, G + 512 * M);
assert!(
budget
.host_line()
.unwrap()
.contains("; what runs counts 7.5G: it uses 1.5G and keeps 6G for growth"),
"{}",
budget.host_line().unwrap()
);
let plan = engine.plan_budget(&engine.admitting(&mut [], None));
assert_eq!(
plan.host,
Some(crate::scheduler::left(9 * G, 7 * G + 512 * M))
);
assert_eq!(
plan.drained.host,
Some(crate::scheduler::left(G + 12 * G - 6 * G, 0)),
"a drain reads the larger margin, nothing of what runs and none of the credit"
);
github.0.lock().unwrap().queued = vec![small(4)];
tight_host(&github, &handle.store, 12 * G);
engine.tick(Shutdown::No).await;
assert_eq!(
registered(&github.calls()).len(),
4,
"lint starts on the host's credit"
);
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let off = measured().replace(
"admission = \"measured\"",
"admission = \"measured\"\npooled_count = false",
);
let (mut engine, handle) = three_settled_jobs(&github, &docker, &dir, &off).await;
github.0.lock().unwrap().queued = vec![small(4)];
tight_host(&github, &handle.store, 12 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(registered(&github.calls()).len(), 3, "off, lint waits");
assert_eq!(
handle.status.borrow().budget.host.as_ref().unwrap().credit,
None
);
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (engine, handle) = three_settled_jobs(&github, &docker, &dir, &measured()).await;
handle.store.flush().await;
drop((engine, handle));
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(32 * G);
engine.recover().await.unwrap();
roomy_readings(&github, &handle.store, 512 * M);
engine.tick(Shutdown::No).await;
first_minute_passes(&mut engine);
github.0.lock().unwrap().queued = vec![small(4)];
tight_host(&github, &handle.store, 12 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(
registered(&github.calls()).len(),
3,
"after a restart, lint waits"
);
assert_eq!(
handle.status.borrow().budget.host.as_ref().unwrap().credit,
None
);
for _ in 0..2 {
tight_host(&github, &handle.store, 16 * G);
engine.tick(Shutdown::No).await;
}
assert_eq!(
registered(&github.calls()).len(),
4,
"once the plain count fits"
);
}
#[tokio::test]
async fn a_requested_reseed_starts_in_the_host_room_plans_see_under_a_host_credit() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = format!(
"{}{}",
measured(),
r#"
[profiles.seedy]
image_stage = "seedy"
cpus = 1
memory = "1G"
slots = 1
[[repositories]]
name = "o/s"
profile = "seedy"
labels = ["self-hosted", "seedy"]
warm = 0
"#
);
let (mut engine, handle) = three_settled_jobs(&github, &docker, &dir, &config).await;
tight_host(&github, &handle.store, 12 * G);
let plan = engine.plan_budget(&engine.admitting(&mut [], None));
assert_eq!(
plan.host,
Some(crate::scheduler::left(9 * G, 7 * G + 512 * M)),
"plans see 1.5G"
);
engine.request_reseed("seedy").unwrap();
tight_host(&github, &handle.store, 12 * G);
engine.tick(Shutdown::No).await;
assert!(
engine.reseeding("seedy"),
"the 1G seed starts in the room plans see"
);
}
#[tokio::test]
async fn a_job_held_with_a_host_credit_waits_for_the_growth_it_keeps() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = three_settled_jobs(&github, &docker, &dir, &measured()).await;
github.0.lock().unwrap().queued = vec![small(4)];
tight_host(&github, &handle.store, 10 * G);
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(registered(&github.calls()).len(), 3);
assert_eq!(
only_reason(&handle),
"The host has 10G of 64G available, but what runs may still grow into it: it uses 1.5G and keeps 6G for \
growth; needs 1.25G, 0B is free"
);
}
#[tokio::test]
async fn a_raised_runner_stays_raised_until_its_job_is_told_apart() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4, "4 × 1.25G");
github.0.lock().unwrap().queued = Vec::new();
let names: Vec<String> = engine
.occupants_mut()
.map(|occupant| occupant.runner_name.clone())
.collect();
for name in names.iter().cloned().chain(["gone".to_owned()]) {
engine.raise(name);
}
engine.note_found(&crate::scheduler::engine::history::Found::named(
names[1].clone(),
77,
));
let key = Key {
repository: "o/r".to_owned(),
workflow: Some("ci".to_owned()),
name: "build".to_owned(),
};
engine.occupants_mut().nth(2).unwrap().set_key(key);
engine.rereserve().await;
let raised: BTreeSet<&str> = engine.raised().iter().map(String::as_str).collect();
assert!(raised.contains(names[0].as_str()), "it still waits");
assert!(
raised.contains(names[1].as_str()),
"its job has no record yet"
);
assert!(!raised.contains(names[2].as_str()), "its job is known");
assert!(!raised.contains("gone"), "no slot holds it");
}
#[tokio::test]
async fn a_runner_raised_past_the_budget_stays_once_a_capture_named_its_job() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = measured_engine(&github, &docker, &dir, G, 0).await;
slice_at(&handle.store, 2 * G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 4, "4 × 1.25G");
engine.memory_max = Some(G);
let names: Vec<String> = engine
.occupants_mut()
.map(|occupant| occupant.runner_name.clone())
.collect();
for name in &names {
engine.raise(name.clone());
}
engine.note_found(&crate::scheduler::engine::history::Found::named(
names[0].clone(),
77,
));
let stops = engine.over_budget(&[]);
assert_eq!(stops.len(), 3, "every runner but the one with a named job");
let first = SlotRef {
profile: "app".to_owned(),
index: 0,
};
assert!(
!stops
.iter()
.any(|stop| matches!(stop, Decision::Stop { slot, .. } if *slot == first)),
"not the runner in slot 1, whose capture named its job"
);
}
#[tokio::test]
async fn a_job_named_without_a_record_makes_its_repository_unknown_though_the_listing_shows_its_runner_idle()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
history_of(&handle.store, "e2e", 2000, 3 * G, 0).await;
runs_of(
&handle.store,
"build",
1000,
2,
200 * M,
0,
Ended::Succeeded,
)
.await;
engine.recover().await.unwrap();
docker.0.lock().unwrap().slot_captures = HashMap::from([(1, "@@job 1 1\n".to_owned())]);
github.0.lock().unwrap().queued = (1..=2).map(job).collect();
slice_at(&handle.store, G);
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
until_state(&handle, 1, JobState::Running).await;
github.0.lock().unwrap().queued = Vec::new();
slice_at(&handle.store, G);
engine.tick(Shutdown::No).await;
let second = engine.occupants_mut().nth(1).unwrap().runner_name.clone();
engine.note_found(&crate::scheduler::engine::history::Found::named(second, 99));
engine.rereserve().await;
let first = engine.occupants_mut().next().unwrap();
assert!(first.has_key(), "job 1 has its record");
assert_eq!(
first.reserved(),
3584 * M,
"the copy goes by its profile's p95, as another of its repository's jobs is unknown"
);
}
#[tokio::test]
async fn a_status_that_cannot_show_the_queue_never_answers_that_nothing_waits_for_the_host() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let answer = |handle: &Handle| handle.status.borrow().host_wait(Some("o/r"));
let (mut engine, handle) = engine_with(&github, &docker, &dir, &measured());
engine.memory_max = Some(10 * G);
engine.publish();
assert_eq!(
answer(&handle),
api::HostWait::Unavailable,
"Docker's recovery has not run"
);
history(&handle.store, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
engine.publish();
assert_eq!(
answer(&handle),
api::HostWait::Unavailable,
"nothing listed yet"
);
slice_at(&handle.store, 0);
host_at(&handle.store, 0, 4 * G + 2 * 1280 * M + 100 * M, 0);
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(answer(&handle), api::HostWait::Waits);
github
.0
.lock()
.unwrap()
.failed_queued
.insert("o/r".to_owned());
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(
answer(&handle),
api::HostWait::Unavailable,
"a failed listing leaves the waiting jobs out"
);
github.0.lock().unwrap().failed_queued.clear();
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(answer(&handle), api::HostWait::Waits, "listed again");
github.0.lock().unwrap().rate_limited =
Some(std::time::SystemTime::now() + Duration::from_secs(60));
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(
answer(&handle),
api::HostWait::Unavailable,
"the rate limit stopped the listing"
);
github.0.lock().unwrap().rate_limited = None;
engine.tick(Shutdown::No).await;
engine.publish();
assert_eq!(
answer(&handle),
api::HostWait::Waits,
"listed once the limit is over"
);
engine.tick(Shutdown::Graceful).await;
engine.publish();
assert_eq!(
answer(&handle),
api::HostWait::Unavailable,
"a stopping daemon lists nothing"
);
}
fn causes(dir: &TestDir) -> Vec<BTreeMap<String, u64>> {
let db = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
let mut statement = db
.prepare("SELECT wait_causes FROM jobs WHERE id <= 4 ORDER BY id")
.unwrap();
statement
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.map(|text| serde_json::from_str(&text.unwrap()).unwrap())
.collect()
}
#[tokio::test]
async fn identical_plans_write_no_cause_and_a_take_or_a_cancel_writes_the_stretch_it_ends() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(
&github,
&docker,
&dir,
&measured().replace("slots = 4", "slots = 2"),
);
engine.memory_max = Some(3 * 1280 * M + 100 * M);
history(&handle.store, G, 0).await;
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = (1..=4).map(job).collect();
slice_at(&handle.store, 0);
let began = std::time::Instant::now();
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()).len(), 2);
tokio::time::sleep(Duration::from_millis(30)).await;
engine.tick(Shutdown::No).await;
handle.store.flush().await;
let settled = causes(&dir);
for _ in 0..2 {
tokio::time::sleep(Duration::from_millis(30)).await;
engine.tick(Shutdown::No).await;
}
handle.store.flush().await;
assert_eq!(
causes(&dir),
settled,
"plans that hold each job for the same cause write nothing"
);
let runner = engine.occupied().next().unwrap().1.runner_name.clone();
engine.note_found(&crate::scheduler::engine::history::Found::named(runner, 1));
github
.0
.lock()
.unwrap()
.queued
.retain(|queued| queued.id != 4);
tokio::time::sleep(Duration::from_millis(30)).await;
engine.tick(Shutdown::No).await;
handle.store.flush().await;
let elapsed = u64::try_from(began.elapsed().as_millis()).unwrap();
let ended = causes(&dir);
let total = |job: &BTreeMap<String, u64>| job.values().sum::<u64>();
for (index, job) in ended.iter().enumerate() {
assert!(
total(job) <= elapsed,
"job {} counts no stretch twice: {job:?} in {elapsed} ms",
index + 1
);
}
assert!(
ended[0].get("booting").is_some_and(|ms| *ms >= 90),
"a taken job counts its wait for GitHub to hand its runner a job: {ended:?}"
);
assert!(
total(&ended[3]) >= 120,
"job 4 counts every stretch until it was cancelled: {ended:?}"
);
}
#[tokio::test]
async fn a_take_the_engine_reads_late_counts_the_jobs_cause_only_up_to_when_its_capture_found_it() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let began = std::time::Instant::now();
let (mut engine, handle) = third_job_waits_for_a_slot(&github, &docker, &dir).await;
tokio::time::sleep(Duration::from_millis(20)).await;
let found_at = time::now_ms();
let until_found = u64::try_from(began.elapsed().as_millis()).unwrap();
tokio::time::sleep(Duration::from_millis(200)).await;
let runner = engine.occupied().next().unwrap().1.runner_name.clone();
engine.note_found(&crate::scheduler::engine::history::Found::named_at(
runner, 3, found_at,
));
handle.store.flush().await;
let third: u64 = causes(&dir)[2].values().sum();
assert!(
third > 0 && third <= until_found,
"job 3's cause counts up to its capture: {third} ms of {until_found}"
);
}
#[tokio::test]
async fn a_stop_writes_every_cause_still_open() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = third_job_waits_for_a_slot(&github, &docker, &dir).await;
engine.tick(Shutdown::No).await;
handle.store.flush().await;
let before: u64 = causes(&dir)[2].values().sum();
tokio::time::sleep(Duration::from_millis(50)).await;
engine.tick(Shutdown::Graceful).await;
handle.store.flush().await;
let after: u64 = causes(&dir)[2].values().sum();
assert!(
after >= before + 50,
"the stop wrote job 3's open cause: {before} ms, then {after} ms"
);
}
#[tokio::test]
async fn a_signals_detach_writes_every_open_cause_after_the_takes_it_was_told_of() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let began = std::time::Instant::now();
let (mut engine, handle) = third_job_waits_for_a_slot(&github, &docker, &dir).await;
tokio::time::sleep(Duration::from_millis(30)).await;
let (found_at, until_found) = (
time::now_ms(),
u64::try_from(began.elapsed().as_millis()).unwrap(),
);
let runner = engine.occupied().next().unwrap().1.runner_name.clone();
engine
.found
.0
.send(crate::scheduler::engine::history::Found::named_at(
runner, 3, found_at,
))
.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
let before_stop = u64::try_from(began.elapsed().as_millis()).unwrap();
handle.control.shutdown(Shutdown::Detach);
tokio::time::timeout(Duration::from_secs(10), engine.schedule())
.await
.unwrap();
handle.store.flush().await;
let causes = causes(&dir);
let total = |job: usize| causes[job].values().sum::<u64>();
assert!(
total(0) >= before_stop - until_found,
"job 1's open cause was written at the stop: {causes:?}"
);
assert!(
total(2) <= until_found,
"job 3 counts up to its take, which the stop read first: {causes:?} in {until_found} ms"
);
}
fn caused(dir: &TestDir, job: u64) -> u64 {
let db = rusqlite::Connection::open(dir.0.join("history.db")).unwrap();
let text: String = db
.query_row("SELECT wait_causes FROM jobs WHERE id = ?1", [job], |row| {
row.get(0)
})
.unwrap();
serde_json::from_str::<BTreeMap<String, u64>>(&text)
.unwrap()
.values()
.sum()
}
#[tokio::test]
async fn a_take_reported_while_a_pass_starts_runners_ends_the_cause_there_even_at_a_checkpoint() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &two_repositories());
engine.recover().await.unwrap();
engine.pause(&one("o/s"), RunningJobs::Keep).await.unwrap();
{
let mut state = github.0.lock().unwrap();
state.queued = vec![job(3)];
state.queued_on.insert(3, "o/s".to_owned());
}
let began = time::now_ms();
engine.tick(Shutdown::No).await;
docker.0.lock().unwrap().latency.start = Duration::from_millis(400);
engine.set_listed("o/r", vec![job(1)]);
engine.checkpoint_due();
let sender = engine.found.0.clone();
let took = tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(100)).await;
let at = time::now_ms();
sender
.send(crate::scheduler::engine::history::Found::named_at(
"elsewhere".to_owned(),
3,
at,
))
.unwrap();
at
});
assert_eq!(engine.refill().await, 1, "job 1's runner starts");
let took_at = took.await.unwrap();
tokio::time::sleep(Duration::from_millis(200)).await;
engine.note_stop_causes();
handle.store.flush().await;
let third = caused(&dir, 3);
assert!(
third > 0 && third <= took_at - began,
"job 3 counts up to its take: {third} ms of {} ms",
took_at - began
);
}