use std::time::SystemTime;
use super::*;
use crate::api::HoldCause;
use crate::scheduler::Hold;
use crate::store::{RunnerRecord, SampleRecord};
use crate::time;
const G: u64 = 1 << 30;
const M: u64 = 1 << 20;
fn pools() -> String {
format!(
"{CONFIG}
[profiles.light]
image_stage = \"app\"
cpus = 1
memory = \"1G\"
slots = 2
[[repositories]]
name = \"o/r\"
profile = \"light\"
labels = [\"r-light\"]
warm = 0
"
)
}
fn light(id: u64) -> QueuedJob {
QueuedJob {
name: "lint".to_owned(),
labels: vec!["r-light".to_owned()],
..job(id)
}
}
fn runners(
engine: &mut Engine<FakeGithub, FakeDocker>,
handle: &Handle,
docker: &FakeDocker,
) -> Vec<Started> {
engine.publish();
let containers = docker.0.lock().unwrap().containers.clone();
handle
.status
.borrow()
.profiles
.iter()
.flat_map(|profile| {
profile.slots.iter().filter_map(|slot| {
let runner = slot.runner.as_ref()?;
let labels = containers
.iter()
.find(|container| container.name == runner.name)
.and_then(|container| container.labels.get("runnerd.labels").cloned())
.unwrap_or_default();
Some(Started {
profile: profile.name.clone(),
labels,
reserved: runner.reserved,
})
})
})
.collect()
}
#[derive(Debug, PartialEq)]
struct Started {
profile: String,
labels: String,
reserved: u64,
}
#[tokio::test]
async fn a_light_job_starts_in_the_light_pool_while_a_heavy_one_waits_for_memory() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &pools());
engine.memory_max = Some(3 * G);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![job(1), light(2)];
engine.tick(Shutdown::No).await;
assert_eq!(
runners(&mut engine, &handle, &docker),
[Started {
profile: "light".to_owned(),
labels: "r-light".to_owned(),
reserved: G,
}]
);
let status = handle.status.borrow().clone();
let heavy = &status.queued[0];
assert_eq!(heavy.profile.as_deref(), Some("app"));
assert!(
matches!(heavy.waiting, Some(api::Waiting::Memory { needs, .. }) if needs == 4 * G),
"the heavy job waits for 4G: {:?}",
heavy.waiting
);
let pools: Vec<(&str, usize)> = status.repositories[0]
.pools
.iter()
.map(|pool| (pool.profile.as_str(), pool.queued_jobs))
.collect();
assert_eq!(pools, [("app", 1), ("light", 1)]);
}
async fn ran_in(store: &Store, profile: &str, labels: &[&str], name: &str, first: u64, peak: u64) {
let at = time::now_ms() - 3600 * 1000;
for id in first..first + 10 {
let runner = format!("earlier-{id}");
store.queued(
"o/r",
&QueuedJob {
created_at: at,
name: name.to_owned(),
labels: labels.iter().map(|label| (*label).to_owned()).collect(),
..job(id)
},
time::now_ms(),
);
store.runner_started(RunnerRecord {
purpose: None,
name: runner.clone(),
repository: "o/r".to_owned(),
profile: profile.to_owned(),
slot: 1,
container: format!("c-{id}"),
started_at: at,
memory_cap: None,
cpu_cap: None,
admission: None,
});
store.claimed(id, 1, &runner, at + 1000);
store.samples(vec![SampleRecord {
runner: runner.clone(),
at: at + 2000,
cpu: 0,
cpu_usage: 0,
memory: peak,
network_received: 0,
network_sent: 0,
cpu_held: crate::observe::metrics::CpuHeld::default(),
events: crate::observe::metrics::Events::default(),
}]);
store.lost(id, at + 60_000);
store.runner_removed(&runner, at + 60_000);
}
store.flush().await;
}
#[tokio::test]
async fn the_light_pool_reserves_only_what_its_own_queued_jobs_predict() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = pools().replace("admission = \"caps\"", "admission = \"measured\"");
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.memory_max = Some(10 * G);
ran_in(
&handle.store,
"app",
&["self-hosted", "app"],
"build",
1000,
3 * G,
)
.await;
ran_in(&handle.store, "light", &["r-light"], "build", 2000, 200 * M).await;
engine.recover().await.unwrap();
handle.store.host_sample(api::HostSample {
at: time::now_ms() / 1000,
cpu: 0,
memory: 2 * G,
});
let light_build = QueuedJob {
name: "build".to_owned(),
..light(2)
};
github.0.lock().unwrap().queued = vec![job(1), light_build];
engine.tick(Shutdown::No).await;
let started = runners(&mut engine, &handle, &docker);
let reserved = |profile: &str| {
started
.iter()
.find(|runner| runner.profile == profile)
.map(|runner| runner.reserved)
};
let (app, light) = (reserved("app").unwrap(), reserved("light").unwrap());
assert!(
app > 3 * G,
"the build's runner reserves past its 3G peak: {app}"
);
assert!(
light < 512 * M,
"the light build's runner reserves for 200M, not the heavy build's 3G: {light}"
);
}
#[tokio::test]
async fn a_job_whose_labels_no_pool_has_says_so_and_starts_nothing() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &pools());
engine.recover().await.unwrap();
let mislabelled = QueuedJob {
labels: vec!["r-lite".to_owned()],
..job(1)
};
let hosted = QueuedJob {
labels: vec!["ubuntu-latest".to_owned()],
..job(2)
};
github.0.lock().unwrap().queued = vec![mislabelled, hosted];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()), Vec::<&String>::new());
engine.publish();
let status = handle.status.borrow().clone();
let waiting = status.queued[0].waiting.clone().unwrap();
assert_eq!(
waiting,
api::Waiting::NoPool {
labels: vec!["r-lite".to_owned()]
}
);
assert_eq!(waiting.kind(), api::WaitKind::Act);
assert_eq!(
waiting.reason(&status.queued[0], &status, 0),
"No profile serves the label r-lite"
);
assert_eq!(
status.queued.len(),
1,
"the GitHub-hosted job is not listed"
);
assert_eq!(status.repositories[0].queued_jobs, 1);
}
#[tokio::test]
async fn pausing_a_repository_pauses_every_pool_of_it() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &pools());
engine.recover().await.unwrap();
engine.pause(&one("o/r"), RunningJobs::Keep).await.unwrap();
github.0.lock().unwrap().queued = vec![job(1), light(2)];
engine.tick(Shutdown::No).await;
assert_eq!(registered(&github.calls()), Vec::<&String>::new());
engine.publish();
let status = handle.status.borrow().clone();
assert_eq!(status.repositories.len(), 1, "one row for the repository");
assert!(
status
.queued
.iter()
.all(|job| job.waiting == Some(api::Waiting::Paused)),
"both jobs wait for the resume: {:?}",
status.queued
);
}
#[tokio::test]
async fn a_pool_whose_runners_fail_holds_back_only_itself() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let config = pools().replacen(
"image_stage = \"app\"\ncpus = 1",
"image_stage = \"lite\"\ncpus = 1",
1,
);
docker.0.lock().unwrap().missing_images = BTreeSet::from(["runnerd-runner:lite".to_owned()]);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![light(1)];
engine.tick(Shutdown::No).await;
github.0.lock().unwrap().queued = vec![light(1), job(2)];
engine.tick(Shutdown::No).await;
let started = runners(&mut engine, &handle, &docker);
assert_eq!(
started
.iter()
.map(|runner| runner.profile.as_str())
.collect::<Vec<_>>(),
["app"]
);
let status = handle.status.borrow().clone();
assert!(
matches!(status.queued[0].waiting, Some(api::Waiting::Backoff { .. })),
"the light job waits for its pool's backoff: {:?}",
status.queued[0].waiting
);
}
#[tokio::test]
async fn a_reload_that_leaves_a_pools_runners_alone_keeps_its_hold_and_streaks() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, _handle) = engine_with(&github, &docker, &dir, &pools());
engine.recover().await.unwrap();
let until = SystemTime::now() + Duration::from_secs(300);
let cause = HoldCause::NoJob;
engine.set_streaks("o/r", "light", 2, 1);
engine.set_held("o/r", "light", Hold { until, cause });
let more_slots = pools().replacen(
"memory = \"1G\"\nslots = 2",
"memory = \"1G\"\nslots = 3",
1,
);
engine.reload(Config::parse(&more_slots).unwrap(), None, None);
let facts = engine.pool_hold("o/r", "light").unwrap();
assert_eq!(
(facts.failures, facts.idle_exits),
(2, 1),
"both streaks stay"
);
assert_eq!(
facts.held.map(|held| (held.until, held.cause)),
Some((until, cause)),
"the hold stays as it was"
);
}
#[tokio::test]
async fn a_start_that_failed_and_left_its_container_holds_its_pool() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, _handle) = engine_with(&github, &docker, &dir, CONFIG);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![job(1)];
docker
.0
.lock()
.unwrap()
.late_failures
.push_back(LateFailure {
takes_job: false,
cleanup_fails: true,
});
engine.tick(Shutdown::No).await;
let facts = engine.pool_hold("o/r", "app").unwrap();
assert_eq!(facts.failures, 1, "it counts as a failed start");
assert_eq!(
facts.held.map(|held| held.cause),
Some(HoldCause::Start),
"the pool is held for the start"
);
}
#[tokio::test]
async fn a_reload_that_changes_a_held_pools_runners_lets_it_start_again_at_once() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let broken = pools().replacen(
"image_stage = \"app\"\ncpus = 1",
"image_stage = \"lite\"\ncpus = 1",
1,
);
docker.0.lock().unwrap().missing_images = BTreeSet::from(["runnerd-runner:lite".to_owned()]);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &broken);
engine.recover().await.unwrap();
github.0.lock().unwrap().queued = vec![light(1)];
engine.tick(Shutdown::No).await;
let backing_off = |engine: &mut Engine<FakeGithub, FakeDocker>, handle: &Handle| {
engine.publish();
matches!(
handle.status.borrow().queued[0].waiting,
Some(api::Waiting::Backoff { .. })
)
};
assert!(
backing_off(&mut engine, &handle),
"the light pool backs off"
);
let more_slots = broken.replacen(
"memory = \"1G\"\nslots = 2",
"memory = \"1G\"\nslots = 3",
1,
);
engine.reload(Config::parse(&more_slots).unwrap(), None, None);
engine.tick(Shutdown::No).await;
assert!(
backing_off(&mut engine, &handle),
"more slots leave the light pool backing off"
);
engine.reload(Config::parse(&pools()).unwrap(), None, None);
engine.tick(Shutdown::No).await;
assert!(
!backing_off(&mut engine, &handle),
"the fixed image ends the light pool's backoff"
);
let profiles: Vec<String> = runners(&mut engine, &handle, &docker)
.into_iter()
.map(|runner| runner.profile)
.collect();
assert_eq!(profiles, ["light"], "the light job's runner starts");
}
#[tokio::test]
async fn a_repository_names_its_first_pool_for_0_0_19_clients_and_a_later_reason_reads_plainly() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("engine"),
);
let (mut engine, handle) = engine_with(&github, &docker, &dir, &pools());
engine.recover().await.unwrap();
engine.publish();
let repository = handle.status.borrow().repositories[0].clone();
assert_eq!(repository.first_profile(), "app");
let later: api::QueuedJob = serde_json::from_value(serde_json::json!({
"id": 1,
"repository": "o/r",
"run_id": 1,
"run_attempt": 1,
"name": "build",
"queued_at": 0,
"waiting": { "kind": "some-later-reason", "detail": 3 },
}))
.unwrap();
let reason = later.waiting.clone().unwrap();
assert_eq!(reason, api::Waiting::Unknown);
assert_eq!(reason.kind(), api::WaitKind::Other);
assert_eq!(
reason.reason(&later, &handle.status.borrow(), 0),
"Waits for a reason this client does not know yet"
);
}