use super::*;
use crate::admission::{Key, Predictor};
use crate::api::HoldCause;
use crate::observe::metrics::{CpuHeld, Events};
use crate::scheduler::{Decision, Hold, PRIORITY_AFTER, SERVED_PRIORITY_AFTER, Wait, Waits};
use crate::sizing::{Measure, Ran};
use crate::store::{RunnerRecord, SampleRecord};
const G: u64 = 1 << 30;
fn big_and_small() -> String {
r#"
[host]
slice = "ci.slice"
isolation = "sysbox"
[caches]
mirror = { enabled = false, memory = "512M", cpus = 1, ttl = "168h" }
pnpm = { max_size = "5G" }
maven = { max_size = "5G" }
[profiles.big]
image_stage = "app"
cpus = 2
memory = "8G"
slots = 2
[profiles.small]
image_stage = "app"
cpus = 1
memory = "1G"
slots = 4
[[repositories]]
name = "o/big"
profile = "big"
labels = ["self-hosted", "big"]
warm = 0
[[repositories]]
name = "o/small"
profile = "small"
labels = ["self-hosted", "small"]
warm = 0
"#
.to_owned()
}
fn big(id: u64) -> QueuedJob {
QueuedJob {
id,
run_id: id,
name: "large".to_owned(),
labels: vec!["self-hosted".to_owned(), "big".to_owned()],
..QueuedJob::default()
}
}
fn small(id: u64) -> QueuedJob {
QueuedJob {
id,
run_id: id,
name: "small".to_owned(),
labels: vec!["self-hosted".to_owned(), "small".to_owned()],
..QueuedJob::default()
}
}
struct Fixture {
github: FakeGithub,
docker: FakeDocker,
engine: Engine<FakeGithub, FakeDocker>,
handle: Handle,
_dir: TestDir,
}
impl Fixture {
async fn new(memory_max: u64) -> Fixture {
Fixture::with(&big_and_small(), memory_max).await
}
async fn with(config: &str, memory_max: u64) -> Fixture {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("priority"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, config);
engine.memory_max = Some(memory_max);
engine.recover().await.unwrap();
Fixture {
github,
docker,
engine,
handle,
_dir: dir,
}
}
fn queue(&self, jobs: Vec<QueuedJob>) {
let mut state = self.github.0.lock().unwrap();
for job in &jobs {
let on = if job.name == "large" {
"o/big"
} else {
"o/small"
};
state.queued_on.insert(job.id, on.to_owned());
}
state.queued = jobs;
}
fn all_busy(&self) {
for runner in &mut self.github.0.lock().unwrap().runners {
runner.busy = true;
}
}
fn wait(&self) -> Option<Wait> {
self.engine
.waits_of("o/big", "big")
.and_then(|waits| waits.memory())
}
fn waits(&self) -> Waits {
self.engine.waits_of("o/big", "big").unwrap_or_default()
}
fn age(&mut self) {
self.age_by(PRIORITY_AFTER);
}
fn age_by(&mut self, by: Duration) {
self.engine
.repositories
.listed_mut("o/big")
.and_then(|state| state.waits_mut("big"))
.and_then(|waits| waits.memory_mut())
.expect("o/big's next job waits")
.since -= by;
}
fn host(&self, room: u64) {
self.handle.store.host_memory(api::HostMemory {
at: crate::time::now_ms() / 1000,
total: 32 * G,
available: room,
slice: 0,
some: Some(api::Pressure::default()),
full: Some(api::Pressure::default()),
holders: Vec::new(),
ceiling: None,
swap: None,
});
}
async fn younger_job_after_a_starved_one(&mut self) {
self.queue(vec![big(10)]);
self.engine.tick(Shutdown::No).await;
self.age();
self.queue(vec![big(10), small(11)]);
self.engine.tick(Shutdown::No).await;
}
fn runners_of(&mut self, repository: &str) -> usize {
self.engine.publish();
self.handle
.status
.borrow()
.profiles
.iter()
.flat_map(|profile| &profile.slots)
.filter_map(|slot| slot.runner.as_ref())
.filter(|runner| runner.repository == repository)
.count()
}
fn reason(&mut self, job: u64) -> Option<api::Waiting> {
self.engine.publish();
self.handle
.status
.borrow()
.queued
.iter()
.find(|queued| queued.id == job)
.and_then(|queued| queued.waiting.clone())
}
async fn starved() -> Fixture {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue((1..=2).map(small).collect());
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 2);
fixture.queue(vec![big(10)]);
fixture.all_busy();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait().map(|wait| wait.job), Some(10));
fixture.age();
fixture
}
async fn served() -> Fixture {
let mut fixture = Fixture::starved().await;
fixture.queue(vec![big(10), big(12)]);
fixture.engine.tick(Shutdown::No).await;
fixture.docker.0.lock().unwrap().containers.clear();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert!(fixture.waits().served(), "the barrier served its pool");
fixture
}
}
#[tokio::test]
async fn reading_the_status_commits_no_wait() {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue((1..=2).map(small).collect());
fixture.engine.tick(Shutdown::No).await;
fixture.all_busy();
fixture.queue(vec![big(10)]);
fixture.engine.set_listed("o/big", vec![big(10)]);
fixture.engine.publish();
assert_eq!(fixture.wait(), None, "a status plan started no wait");
}
#[tokio::test]
async fn a_pass_between_ticks_commits_a_wait_from_the_listing_it_plans_from() {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue((1..=3).map(small).collect());
fixture.engine.tick(Shutdown::No).await;
fixture.all_busy();
fixture.engine.tick(Shutdown::No).await;
fixture.engine.set_listed("o/big", vec![big(10)]);
fixture.engine.listing.listing += 1;
fixture.docker.0.lock().unwrap().containers.remove(0);
fixture.engine.exits.any();
fixture.engine.turnover().await;
assert_eq!(
fixture.wait().map(|wait| wait.job),
Some(10),
"the pass held job 10 for memory"
);
}
#[tokio::test]
async fn younger_jobs_stop_taking_memory_and_the_starved_pool_starts_once_it_drained() {
let mut fixture = Fixture::starved().await;
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(
fixture.reason(11),
Some(api::Waiting::BehindMemory { job: 10 })
);
assert_eq!(
fixture.runners_of("o/small"),
2,
"no younger runner started"
);
fixture.docker.0.lock().unwrap().containers.clear();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert_eq!(
fixture.wait(),
None,
"its runner in its slot ended the wait"
);
}
#[tokio::test]
async fn a_start_that_fails_preparing_its_caches_keeps_the_wait() {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue(vec![big(10)]);
fixture.engine.set_memory_wait(
"o/big",
"big",
Wait {
job: 10,
since: std::time::SystemTime::now() - PRIORITY_AFTER,
},
);
fixture.docker.0.lock().unwrap().fail_volumes = 1;
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 0, "no runner went in its slot");
assert_eq!(fixture.wait().map(|wait| wait.job), Some(10));
}
#[tokio::test]
async fn a_failed_listing_keeps_the_wait_and_suspends_the_barrier() {
let mut fixture = Fixture::starved().await;
let wait = fixture.wait();
fixture
.github
.0
.lock()
.unwrap()
.failed_queued
.insert("o/big".to_owned());
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait(), wait, "its age is kept");
assert_eq!(fixture.runners_of("o/small"), 3, "younger jobs went on");
fixture.github.0.lock().unwrap().failed_queued.clear();
fixture.all_busy();
fixture.queue(vec![big(10), small(12), small(13)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(
fixture.reason(13),
Some(api::Waiting::BehindMemory { job: 10 }),
"the same job holds again on the next listing that succeeds"
);
}
#[tokio::test]
async fn a_stop_ends_no_wait() {
let mut fixture = Fixture::starved().await;
let wait = Wait {
job: 12,
since: fixture.wait().unwrap().since,
};
fixture.engine.set_memory_wait("o/big", "big", wait);
fixture.engine.tick(Shutdown::Graceful).await;
assert_eq!(fixture.wait(), Some(wait));
}
#[tokio::test]
async fn a_job_gone_from_a_listing_that_succeeded_ends_the_wait_and_the_barrier() {
let mut fixture = Fixture::starved().await;
fixture.queue(vec![small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait(), None);
assert_eq!(fixture.runners_of("o/small"), 3);
}
#[tokio::test]
async fn pausing_the_starved_repository_ends_the_wait() {
let mut fixture = Fixture::starved().await;
fixture
.engine
.pause(&one("o/big"), RunningJobs::Keep)
.await
.unwrap();
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait(), None);
assert_eq!(fixture.runners_of("o/small"), 3);
}
#[tokio::test]
async fn a_pool_backing_off_after_failed_runners_loses_its_wait() {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue(vec![big(10)]);
fixture.engine.set_memory_wait(
"o/big",
"big",
Wait {
job: 10,
since: std::time::SystemTime::now() - PRIORITY_AFTER,
},
);
fixture.github.0.lock().unwrap().register_fails = true;
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait(), None);
fixture.github.0.lock().unwrap().register_fails = false;
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 1, "younger jobs go on");
}
#[tokio::test]
async fn a_reload_that_removes_the_pool_drops_its_wait() {
let mut fixture = Fixture::starved().await;
let without = big_and_small().replace(
"[[repositories]]\nname = \"o/big\"\nprofile = \"big\"\nlabels = [\"self-hosted\", \"big\"]\nwarm = 0\n",
"[[repositories]]\nname = \"o/big\"\nprofile = \"small\"\nlabels = [\"self-hosted\", \"big\"]\nwarm = 0\n",
);
fixture
.engine
.reload(Config::parse(&without).unwrap(), Some(9 * G), None);
assert!(
fixture.engine.keeps_no_waits("o/big"),
"the pool in `big` is gone"
);
}
#[tokio::test]
async fn a_reload_that_changes_the_pools_runners_keeps_its_wait() {
let mut fixture = Fixture::starved().await;
let before = fixture.wait();
let more_cpus = big_and_small().replace("cpus = 2\n", "cpus = 3\n");
fixture
.engine
.reload(Config::parse(&more_cpus).unwrap(), Some(9 * G), None);
assert_eq!(
fixture.wait(),
before,
"job 10 has waited as long as before"
);
}
#[tokio::test]
async fn a_reload_that_makes_the_runner_past_the_slice_lifts_the_barrier() {
let mut fixture = Fixture::starved().await;
let larger = big_and_small().replace("memory = \"8G\"", "memory = \"10G\"");
fixture
.engine
.reload(Config::parse(&larger).unwrap(), Some(9 * G), None);
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 3, "no drain could serve it");
assert!(
matches!(fixture.reason(10), Some(api::Waiting::Memory { .. })),
"{:?}",
fixture.reason(10)
);
}
#[tokio::test]
async fn a_runner_that_fits_the_slice_but_not_beside_the_mirror_erects_no_barrier() {
let config = big_and_small().replace(
"mirror = { enabled = false, memory = \"512M\"",
"mirror = { enabled = true, memory = \"2G\"",
);
let mut fixture = Fixture::with(&config, 9 * G).await;
fixture.queue(vec![big(10)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait().map(|wait| wait.job), Some(10));
fixture.age();
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 1, "younger jobs go on");
}
#[tokio::test]
async fn a_runner_that_fits_only_a_slice_with_no_runner_erects_no_barrier_while_a_warm_one_stays() {
let config = big_and_small()
+ r#"
[[repositories]]
name = "o/warm"
profile = "small"
labels = ["self-hosted", "warmed"]
warm = 1
"#;
let mut fixture = Fixture::with(&config, 24 * G).await;
let lately = QueuedJob {
id: 1,
run_id: 1,
name: "warmed".to_owned(),
labels: vec!["self-hosted".to_owned(), "warmed".to_owned()],
created_at: crate::time::now_ms() - 300_000,
..QueuedJob::default()
};
fixture
.handle
.store
.queued("o/warm", &lately, crate::time::now_ms());
fixture.handle.store.flush().await;
fixture.host(21 * G / 2);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/warm"), 1, "a 1G warm runner runs");
fixture.engine.set_host_short();
fixture.younger_job_after_a_starved_one().await;
assert!(
matches!(fixture.reason(10), Some(api::Waiting::HostMemory { .. })),
"{:?}",
fixture.reason(10)
);
assert_eq!(fixture.runners_of("o/small"), 1, "younger jobs go on");
assert_eq!(fixture.runners_of("o/warm"), 1, "its room would not do");
assert!(
fixture.engine.host_is_short(),
"beside the warm runner it fits no margin, so the larger one stands"
);
}
#[tokio::test]
async fn on_a_steady_reading_a_job_between_the_margins_starts_and_the_margin_and_barrier_stay_put()
{
let config = big_and_small().replace("memory = \"1G\"", "memory = \"512M\"");
let mut fixture = Fixture::with(&config, 24 * G).await;
fixture.host(21 * G / 2);
fixture.queue(vec![big(9)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
fixture.all_busy();
fixture.queue(Vec::new());
fixture.engine.tick(Shutdown::No).await;
fixture.queue(vec![big(10)]);
fixture.engine.tick(Shutdown::No).await;
assert!(fixture.engine.host_is_short(), "job 10 fits no margin");
fixture.age();
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert!(
!fixture.engine.host_is_short(),
"job 11 fits the plain margin, so the larger one goes"
);
assert_eq!(
fixture.runners_of("o/small"),
0,
"this plan had the larger one"
);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 1, "the next plan has job 11");
fixture.queue(vec![big(10), small(11), small(12)]);
for tick in 0..6 {
fixture.engine.tick(Shutdown::No).await;
assert!(
fixture.engine.host_is_short(),
"tick {tick}: job 10 fits no margin, so the larger one stands"
);
assert!(
matches!(fixture.reason(10), Some(api::Waiting::HostMemory { .. })),
"tick {tick}: {:?}",
fixture.reason(10)
);
assert!(
matches!(fixture.reason(12), Some(api::Waiting::HostMemory { .. })),
"tick {tick}: no barrier stands: {:?}",
fixture.reason(12)
);
}
assert!(fixture.wait().is_some(), "job 10 keeps its place");
assert_eq!(fixture.runners_of("o/small"), 1, "nothing else fits");
}
#[tokio::test]
async fn a_job_the_slice_holds_keeps_its_barrier_with_a_drain_between_the_host_margins() {
let mut fixture = Fixture::starved().await;
fixture.host(21 * G / 2);
fixture.queue(vec![big(10), small(11)]);
for tick in 0..4 {
fixture.engine.tick(Shutdown::No).await;
assert!(
matches!(fixture.reason(10), Some(api::Waiting::Memory { .. })),
"tick {tick}: {:?}",
fixture.reason(10)
);
assert_eq!(
fixture.reason(11),
Some(api::Waiting::BehindMemory { job: 10 }),
"tick {tick}: the barrier stands"
);
assert!(
!fixture.engine.host_is_short(),
"tick {tick}: no job waits for the host"
);
}
assert_eq!(fixture.runners_of("o/small"), 2, "job 11 waits behind it");
}
#[tokio::test]
async fn a_host_too_short_for_the_runner_after_a_full_drain_erects_no_barrier() {
let mut fixture = Fixture::new(24 * G).await;
fixture.host(7 * G);
fixture.younger_job_after_a_starved_one().await;
assert!(
matches!(fixture.reason(10), Some(api::Waiting::HostMemory { .. })),
"{:?}",
fixture.reason(10)
);
assert_eq!(fixture.runners_of("o/small"), 1, "younger jobs go on");
}
#[tokio::test]
async fn a_runner_that_fits_the_slice_but_not_beside_a_seed_container_erects_no_barrier() {
let mut fixture = Fixture::new(9 * G).await;
fixture.engine.seed_runs("retired", 2 * G);
fixture.younger_job_after_a_starved_one().await;
assert_eq!(fixture.runners_of("o/small"), 1, "younger jobs go on");
}
#[tokio::test]
async fn a_runner_that_fits_the_slice_but_not_beside_a_shell_erects_no_barrier() {
let config = big_and_small().replace("slots = 4", "slots = 4\nrecord_failures = true");
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("priority"),
);
let mut image = records::record_image(99, 60);
image
.labels
.insert("runnerd.profile".to_owned(), "small".to_owned());
docker.0.lock().unwrap().images = vec![image];
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(17 * G / 2);
engine.recover().await.unwrap();
engine
.open_shell(99, docker::TtySize { cols: 80, rows: 24 })
.await
.unwrap();
let mut fixture = Fixture {
github,
docker,
engine,
handle,
_dir: dir,
};
fixture.younger_job_after_a_starved_one().await;
assert_eq!(fixture.runners_of("o/small"), 1, "younger jobs go on");
}
#[tokio::test]
async fn a_runner_not_yet_named_leaves_the_oldest_jobs_wait() {
let mut fixture = Fixture::new(17 * G).await;
fixture.queue(vec![big(9)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
let wait = Wait {
job: 10,
since: std::time::SystemTime::now() - PRIORITY_AFTER,
};
fixture.engine.set_memory_wait("o/big", "big", wait);
fixture.queue(vec![big(10)]);
fixture.all_busy();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait(), Some(wait), "its age is kept");
}
#[tokio::test]
async fn a_restart_starts_the_wait_again() {
let fixture = Fixture::starved().await;
let (github, docker) = (fixture.github.clone(), fixture.docker.clone());
let dir = TestDir::new("restarted");
let (mut engine, _handle) = engine_with(&github, &docker, &dir, &big_and_small());
engine.memory_max = Some(9 * G);
engine.recover().await.unwrap();
fixture.queue(vec![big(10), small(11)]);
engine.tick(Shutdown::No).await;
let wait = engine
.waits_of("o/big", "big")
.and_then(|waits| waits.memory())
.expect("the first plan held it again");
assert!(
wait.since.elapsed().unwrap_or_default() < PRIORITY_AFTER,
"not carried over"
);
}
#[tokio::test]
async fn a_pools_first_job_holds_only_after_the_first_limit() {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue((1..=2).map(small).collect());
fixture.engine.tick(Shutdown::No).await;
fixture.all_busy();
fixture.queue(vec![big(10)]);
fixture.engine.tick(Shutdown::No).await;
fixture.age_by(PRIORITY_AFTER.saturating_sub(Duration::from_secs(5)));
fixture.queue(vec![big(10), small(11)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 3, "the job goes by it");
fixture.age_by(Duration::from_secs(5));
fixture.all_busy();
fixture.queue(vec![big(10), small(12), small(13)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(
fixture.reason(13),
Some(api::Waiting::BehindMemory { job: 10 })
);
}
#[tokio::test]
async fn once_a_barrier_served_a_pool_its_next_job_holds_after_the_short_limit() {
let mut fixture = Fixture::served().await;
assert_eq!(fixture.wait().map(|wait| wait.job), Some(12));
fixture.age_by(SERVED_PRIORITY_AFTER.saturating_sub(Duration::from_secs(5)));
fixture.all_busy();
fixture.queue(vec![big(12), small(13)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 1, "not yet");
fixture.age_by(Duration::from_secs(5));
fixture.all_busy();
fixture.queue(vec![big(12), small(14), small(15)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(
fixture.reason(15),
Some(api::Waiting::BehindMemory { job: 12 })
);
}
#[tokio::test]
async fn the_mark_stays_through_a_failed_listing_and_goes_when_a_listing_shows_the_queue_empty() {
let mut fixture = Fixture::served().await;
fixture.all_busy();
let failed = |fixture: &Fixture| {
let mut state = fixture.github.0.lock().unwrap();
state.failed_queued.insert("o/big".to_owned());
};
failed(&fixture);
fixture.queue(Vec::new());
fixture.engine.tick(Shutdown::No).await;
assert!(fixture.waits().served(), "a failed listing says nothing");
fixture.github.0.lock().unwrap().failed_queued.clear();
fixture.queue(vec![big(13)]);
fixture.engine.tick(Shutdown::No).await;
assert!(
fixture.waits().served(),
"job 12 went elsewhere, 13 is queued"
);
fixture.queue(Vec::new());
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.waits(), Waits::default(), "the queue emptied");
}
#[tokio::test]
async fn the_mark_outlives_the_wait_it_came_from() {
let mut fixture = Fixture::starved().await;
fixture.queue(vec![big(10)]);
fixture.engine.tick(Shutdown::No).await;
fixture.docker.0.lock().unwrap().containers.clear();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert_eq!(fixture.wait(), None, "nothing waits");
assert!(fixture.waits().served(), "job 10 is listed still");
fixture.queue(vec![big(12)]);
fixture.engine.tick(Shutdown::No).await;
assert!(fixture.waits().served(), "the queue never emptied");
}
#[tokio::test]
async fn a_barrier_whose_job_went_elsewhere_served_nobody() {
let mut fixture = Fixture::starved().await;
fixture.queue(vec![big(10), big(12)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.waits().owner(), Some(10));
fixture.queue(vec![big(12)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.waits().owner(), None);
fixture.docker.0.lock().unwrap().containers.clear();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert!(!fixture.waits().served(), "job 12 owned no barrier");
let mut fixture = Fixture::starved().await;
fixture.queue(vec![big(10), big(12)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.waits().owner(), Some(10));
fixture.queue(vec![big(12)]);
fixture.docker.0.lock().unwrap().containers.clear();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert!(!fixture.waits().served(), "the same tick");
}
#[tokio::test]
async fn a_start_no_barrier_came_before_serves_nobody() {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue(vec![big(10)]);
fixture.engine.set_memory_wait(
"o/big",
"big",
Wait {
job: 10,
since: std::time::SystemTime::now() - PRIORITY_AFTER,
},
);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert_eq!(fixture.waits(), Waits::default());
}
#[tokio::test]
async fn only_a_runner_in_its_slot_serves_the_pool() {
let mut fixture = Fixture::starved().await;
fixture.queue(vec![big(10), big(12)]);
fixture.engine.tick(Shutdown::No).await;
fixture.docker.0.lock().unwrap().containers.clear();
fixture.docker.0.lock().unwrap().fail_volumes = 1;
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 0, "no runner went in its slot");
assert!(!fixture.waits().served());
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/big"), 1);
assert!(fixture.waits().served(), "the retry was for the same owner");
}
#[tokio::test]
async fn pausing_backing_off_or_preparing_takes_the_mark() {
async fn stopped(stop: impl FnOnce(&mut Fixture)) {
let mut fixture = Fixture::served().await;
stop(&mut fixture);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.waits(), Waits::default(), "nothing is left behind");
}
let mut paused = Fixture::served().await;
paused
.engine
.pause(&one("o/big"), RunningJobs::Keep)
.await
.unwrap();
paused.engine.tick(Shutdown::No).await;
assert_eq!(paused.waits(), Waits::default(), "pausing");
stopped(|fixture| {
fixture.engine.set_held(
"o/big",
"big",
Hold {
until: std::time::SystemTime::now() + Duration::from_secs(60),
cause: HoldCause::NoJob,
},
);
})
.await;
stopped(|fixture| {
fixture.engine.ask_reseed("big");
})
.await;
}
#[tokio::test]
async fn a_runner_that_connected_and_left_takes_no_mark() {
let mut fixture = Fixture::served().await;
{
let mut docker = fixture.docker.0.lock().unwrap();
docker.runner_log = Some("2026-11-07 13:29:01Z: Listening for Jobs\n".to_owned());
for container in &mut docker.containers {
if container.labels["runnerd.repository"] == "o/big" {
container.running = false;
}
}
}
fixture.engine.tick(Shutdown::No).await;
let hold = fixture.engine.pool_hold("o/big", "big").unwrap();
assert!(hold.held.is_none(), "no backoff: {hold:?}");
assert!(fixture.waits().served(), "the mark stands");
}
#[tokio::test]
async fn a_reload_that_removes_the_pool_takes_the_mark() {
let mut fixture = Fixture::served().await;
let moved = big_and_small().replace(
"[[repositories]]\nname = \"o/big\"\nprofile = \"big\"\nlabels = [\"self-hosted\", \"big\"]\nwarm = 0\n",
"[[repositories]]\nname = \"o/big\"\nprofile = \"small\"\nlabels = [\"self-hosted\", \"big\"]\nwarm = 0\n",
);
fixture
.engine
.reload(Config::parse(&moved).unwrap(), Some(9 * G), None);
assert_eq!(fixture.waits(), Waits::default());
}
#[tokio::test]
async fn a_restart_loses_the_mark() {
let fixture = Fixture::served().await;
let (github, docker) = (fixture.github.clone(), fixture.docker.clone());
let dir = TestDir::new("restarted");
let (mut engine, _handle) = engine_with(&github, &docker, &dir, &big_and_small());
engine.memory_max = Some(9 * G);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
let waits = engine.waits_of("o/big", "big");
assert!(!waits.is_some_and(|waits| waits.served()));
}
fn named(id: u64, name: &str) -> QueuedJob {
QueuedJob {
name: name.to_owned(),
..small(id)
}
}
fn rfc3339(ms: u64) -> String {
let (days, secs) = (ms / 86_400_000, ms / 1000 % 86_400);
let shifted = days + 719_468;
let era = shifted / 146_097;
let of_era = shifted % 146_097;
let year_of_era = (of_era - of_era / 1460 + of_era / 36_524 - of_era / 146_096) / 365;
let of_year = of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
let month_index = (5 * of_year + 2) / 153;
let day = of_year - (153 * month_index + 2) / 5 + 1;
let month = if month_index < 10 {
month_index + 3
} else {
month_index - 9
};
let year = year_of_era + era * 400 + u64::from(month <= 2);
let (hour, minute, second) = (secs / 3600, secs / 60 % 60, secs % 60);
format!("{year}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z")
}
impl Fixture {
async fn ran(&self, name: &str, first: u64, count: u64, minutes: u64) {
let store = &self.handle.store;
let at = crate::time::now_ms() - 24 * 3600 * 1000;
for id in first..first + count {
let runner = format!("earlier-{id}");
let job = QueuedJob {
created_at: at,
..named(id, name)
};
store.queued("o/small", &job, at);
store.runner_started(RunnerRecord {
purpose: None,
name: runner.clone(),
repository: "o/small".to_owned(),
profile: "small".to_owned(),
slot: 1,
container: format!("c-{id}"),
started_at: at,
memory_cap: Some(G),
cpu_cap: Some(1000),
admission: None,
});
store.claimed(id, id, &runner, at + 1000);
store.samples(vec![SampleRecord {
runner: runner.clone(),
at: at + 2000,
cpu: 0,
cpu_usage: 0,
memory: G / 2,
network_received: 0,
network_sent: 0,
cpu_held: CpuHeld::default(),
events: Events {
oom: Some(0),
oom_kill: Some(0),
sock_throttled: None,
local_oom: None,
},
}]);
store.details(JobDetails {
id,
run_id: id,
run_attempt: 1,
name: name.to_owned(),
workflow_name: None,
head_branch: None,
html_url: None,
labels: Vec::new(),
created_at: None,
status: "completed".to_owned(),
conclusion: Some("success".to_owned()),
started_at: None,
completed_at: Some(rfc3339(at + minutes * 60_000)),
runner_name: Some(runner.clone()),
steps: Vec::new(),
});
store.runner_removed(&runner, at + minutes * 60_000);
}
store.flush().await;
}
fn running(&mut self, name: &str) {
for occupant in self.engine.occupants_mut() {
occupant.set_key(Key {
repository: "o/small".to_owned(),
workflow: None,
name: name.to_owned(),
});
}
self.engine.forget_predictor();
}
async fn behind_the_barrier(name: &str) -> Fixture {
let mut fixture = Fixture::starved().await;
fixture.ran("small", 100, 3, 30).await;
fixture.ran("quick", 200, 3, 2).await;
fixture.ran("rare", 300, 2, 2).await;
fixture.ran("slow", 400, 3, 45).await;
fixture.running("small");
fixture.queue(vec![big(10), named(11, name)]);
fixture
}
}
#[tokio::test]
async fn a_job_that_usually_ends_before_the_room_gathers_starts_behind_the_barrier() {
let mut fixture = Fixture::behind_the_barrier("quick").await;
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 3, "job 11's runner started");
assert_eq!(fixture.runners_of("o/big"), 0);
assert!(
matches!(fixture.reason(10), Some(api::Waiting::Memory { .. })),
"the owner keeps its reason"
);
assert_eq!(
fixture.wait().map(|wait| wait.job),
Some(10),
"a start behind the barrier ends its own pool's wait, not the owner's"
);
}
#[tokio::test]
async fn a_runner_that_passed_counts_against_the_room_behind_the_barrier_until_it_goes() {
let mut fixture = Fixture::behind_the_barrier("quick").await;
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 3, "job 11 passed");
fixture.queue(vec![big(10), named(11, "quick"), named(12, "quick")]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 3, "job 12 waits");
assert_eq!(
fixture.reason(12),
Some(api::Waiting::BehindMemory { job: 10 })
);
}
#[tokio::test]
async fn a_job_that_would_outlast_the_room_or_has_too_few_runs_stays_behind_the_barrier() {
for name in ["slow", "rare"] {
let mut fixture = Fixture::behind_the_barrier(name).await;
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 2, "{name} stays queued");
assert_eq!(
fixture.reason(11),
Some(api::Waiting::BehindMemory { job: 10 }),
"{name} says what it says behind a strict barrier"
);
}
}
#[tokio::test]
async fn a_pool_with_short_and_long_jobs_queued_is_judged_by_the_long_one() {
let mut fixture = Fixture::behind_the_barrier("quick").await;
fixture.queue(vec![big(10), named(11, "quick"), named(12, "slow")]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.runners_of("o/small"), 2);
assert_eq!(
fixture.reason(11),
Some(api::Waiting::BehindMemory { job: 10 })
);
}
async fn a_runner_not_named_hides(name: &str) -> Fixture {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue((1..=3).map(small).collect());
fixture.engine.tick(Shutdown::No).await;
fixture.queue(vec![big(10)]);
fixture.all_busy();
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait().map(|wait| wait.job), Some(10));
fixture.age();
fixture.ran("small", 100, 3, 30).await;
fixture.ran("quick", 200, 3, 2).await;
fixture.ran("slow", 400, 3, 45).await;
fixture.running("small");
fixture.queue(vec![big(10), named(11, name), named(12, "quick")]);
let unnamed = fixture
.engine
.occupants_mut()
.find(|occupant| occupant.repository == "o/small")
.unwrap();
unnamed.engagement.unname();
fixture.engine.tick(Shutdown::No).await;
fixture
}
#[tokio::test]
async fn a_job_hidden_for_a_runner_not_yet_named_counts_towards_its_pools_longest() {
let mut short = a_runner_not_named_hides("quick").await;
assert_eq!(short.runners_of("o/small"), 4, "a short job passes");
let mut long = a_runner_not_named_hides("slow").await;
assert_eq!(long.runners_of("o/small"), 3, "job 12 stays behind");
}
async fn a_pass_behind_the_barrier(name: &str) -> Fixture {
let mut fixture = Fixture::new(9 * G).await;
fixture.queue((1..=3).map(small).collect());
fixture.engine.tick(Shutdown::No).await;
fixture.all_busy();
fixture.queue((1..=3).map(small).chain([big(10)]).collect());
fixture.engine.tick(Shutdown::No).await;
fixture.queue(vec![big(10)]);
fixture.engine.tick(Shutdown::No).await;
assert_eq!(fixture.wait().map(|wait| wait.job), Some(10));
fixture.age();
fixture.ran("small", 100, 3, 30).await;
fixture.ran("quick", 200, 3, 2).await;
fixture.ran("slow", 400, 3, 45).await;
fixture.running("small");
fixture.engine.refresh_predictor().await;
fixture
.engine
.set_listed("o/small", vec![named(11, name), named(12, name)]);
fixture.engine.listing.listing += 1;
fixture.docker.0.lock().unwrap().containers.remove(0);
fixture.engine.exits.any();
fixture.engine.turnover().await;
fixture
}
#[tokio::test]
async fn a_pass_between_ticks_goes_by_the_same_rule_as_a_tick() {
let mut passed = a_pass_behind_the_barrier("quick").await;
assert_eq!(passed.runners_of("o/small"), 3, "two left, and job 12's");
let mut held = a_pass_behind_the_barrier("slow").await;
assert_eq!(held.runners_of("o/small"), 2, "two left");
assert_eq!(
held.reason(12),
Some(api::Waiting::BehindMemory { job: 10 })
);
}
#[tokio::test]
async fn under_caps_a_history_changes_no_reservation() {
let config = big_and_small().replace("[caches]", "cpu_overcommit = 1.0\n\n[caches]");
let mut fixture = Fixture::with(&config, 9 * G).await;
fixture.engine.cpus = Some(8000);
let run = Ran {
repository: "o/small".to_owned(),
workflow: None,
name: "small".to_owned(),
profile: "small".to_owned(),
memory_cap: Some(G),
cpu_cap: Some(1000),
memory_peak: G / 2,
cpu_peak: 200,
cpu_average: Some(100),
queued_at: 0,
started_at: Some(0),
runner_started_at: 0,
finished_at: Some(60_000),
oom_killed: false,
hit_cap: Some(false),
own_cap: Some(false),
measure: Measure::WithoutPageCache,
succeeded: true,
failed: false,
run_id: 1,
run_attempt: 1,
branch: None,
reserved: None,
host_waited: 0,
unseen: 0,
cpu_periods: None,
cpu_throttled: None,
slot_waited: 0,
};
let mut predictor = Predictor::new(&fixture.engine.config.sizing);
for _ in 0..20 {
predictor.learn(&run, "small", G, 60_000);
}
fixture
.engine
.set_predictor(predictor, std::time::SystemTime::now());
fixture.queue(vec![small(1)]);
fixture.engine.tick(Shutdown::No).await;
let (plan, _) = fixture.engine.plan();
assert_eq!(plan.decisions, [] as [Decision; 0]);
assert_eq!(
fixture.engine.cpu_in_use(),
1000,
"the runner's whole 1 CPU"
);
}