use std::sync::atomic::AtomicBool;
use super::*;
use crate::api::Stopping;
use crate::scheduler::engine::STOP_DEADLINE;
fn busy_and_idle(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
busy_and_idle_with(github, docker, dir, CONFIG)
}
fn busy_and_idle_with(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
config: &str,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
running_runner(github, docker, "@@job 1 1\n");
let idle = identity("ci", ID_A).runner_name("app", 2, 1);
{
let mut state = github.0.lock().unwrap();
state.jobs.insert(1, details(1, "in_progress", None));
state.runners.push(runner(8, &idle, false));
}
{
let mut state = docker.0.lock().unwrap();
state.containers.push(runner_container(2, 8, true));
state
.container_captures
.insert("container-2".to_owned(), String::new());
state
.started
.insert("container-2".to_owned(), crate::time::now_ms());
let mirror = Names::new("runnerd").mirror();
state.containers.push(Container {
id: mirror.clone(),
name: mirror,
running: true,
labels: HashMap::from([("runnerd.mirror".to_owned(), String::new())]),
});
}
let config = config.replace("mirror = { enabled = false", "mirror = { enabled = true");
engine_with(github, docker, dir, &config)
}
fn container_names(docker: &FakeDocker) -> BTreeSet<String> {
docker
.0
.lock()
.unwrap()
.containers
.iter()
.map(|container| container.name.clone())
.collect()
}
async fn next_start(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> (Vec<Option<String>>, Handle) {
let config = CONFIG.replace("mirror = { enabled = false", "mirror = { enabled = true");
let (mut engine, handle) = engine_with(github, docker, dir, &config);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
engine.publish();
let slots = handle.status.borrow().profiles[0]
.slots
.iter()
.map(|slot| slot.runner.as_ref().map(|runner| runner.name.clone()))
.collect();
(slots, handle)
}
#[tokio::test]
async fn a_signal_removes_idle_runners_and_leaves_busy_ones_and_the_mirror_to_the_next_start() {
for took_job in [false, true] {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = busy_and_idle(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github
.until_calls(
|calls| calls.iter().any(|call| call.starts_with("runners ")),
"asked for its runners",
)
.await;
if took_job {
github.0.lock().unwrap().busy_deletes = 1;
}
let (github_calls, docker_calls) = (github.calls().len(), docker.calls().len());
handle.control.shutdown(Shutdown::Detach);
tokio::time::timeout(Duration::from_secs(10), run)
.await
.expect("a signal stops at once")
.unwrap();
let name = |slot| identity("ci", ID_A).runner_name("app", slot, 1);
let mut kept = BTreeSet::from([name(1), Names::new("runnerd").mirror()]);
if took_job {
kept.insert(name(2));
}
assert_eq!(container_names(&docker), kept, "took a job: {took_job}");
let deleted: Vec<String> = github.calls()[github_calls..]
.iter()
.filter(|call| call.starts_with("delete "))
.cloned()
.collect();
assert_eq!(
deleted,
["delete 8"],
"only the idle runner is deregistered"
);
let removed: Vec<String> = docker.calls()[docker_calls..]
.iter()
.filter(|call| call.starts_with("remove "))
.cloned()
.collect();
let expected: &[&str] = if took_job {
&[]
} else {
&["remove container-2"]
};
assert_eq!(removed, expected, "took a job: {took_job}");
let (slots, handle) = next_start(&github, &docker, &dir).await;
assert_eq!(
slots,
[Some(name(1)), took_job.then(|| name(2))],
"the next start adopts what runs (took a job: {took_job})"
);
assert_eq!(registered(&github.calls()), Vec::<&String>::new());
handle.store.flush().await;
let job = handle.store.job(1).await.unwrap().unwrap();
assert_eq!(job.summary.state, JobState::Running);
}
}
#[tokio::test]
async fn status_says_a_signal_leaves_running_jobs_while_it_removes_idle_runners() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = busy_and_idle(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github
.until_calls(
|calls| calls.iter().any(|call| call.starts_with("runners ")),
"asked for its runners",
)
.await;
github.hold_deletes();
handle.control.shutdown(Shutdown::Detach);
github.until_waiting(1).await;
assert_eq!(
handle.status.borrow().stopping,
Some(Stopping::LeavesJobs),
"status shows the stop while the idle runner is removed"
);
github.release_deletes();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap();
}
#[tokio::test]
async fn a_stop_ends_when_docker_is_gone_or_fails_midway_and_the_next_start_takes_over() {
for gone in [true, false] {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = busy_and_idle(&github, &docker, &dir);
if gone {
docker
.0
.lock()
.unwrap()
.containers
.retain(|container| container.id != "container-2");
github
.0
.lock()
.unwrap()
.runners
.retain(|runner| runner.id != 8);
}
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github
.until_calls(
|calls| calls.iter().any(|call| call.starts_with("runners ")),
"asked for its runners",
)
.await;
if gone {
docker.set_outage(Outage::Refused);
} else {
docker.0.lock().unwrap().remove_outage = Outage::Refused;
}
handle.control.shutdown(Shutdown::Graceful);
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap_or_else(|_| panic!("the stop never ended (Docker gone: {gone})"))
.unwrap();
assert_eq!(
handle.shutdown(Shutdown::Graceful),
Some(Stopping::LeavesJobs),
"a second `runnerd stop` says what the first does once Docker stopped answering (Docker gone: {gone})"
);
assert_eq!(handle.shutdown(Shutdown::Force), Some(Stopping::KillsJobs));
let calls = github.calls();
for id in [7, 8] {
let deletes = calls
.iter()
.filter(|call| **call == format!("delete {id}"))
.count();
assert!(deletes <= 1, "runner {id} deleted {deletes} times");
}
docker.set_outage(Outage::None);
docker.0.lock().unwrap().remove_outage = Outage::None;
let (slots, handle) = next_start(&github, &docker, &dir).await;
assert_eq!(
slots[0],
Some(identity("ci", ID_A).runner_name("app", 1, 1)),
"job 1's runner is adopted (Docker gone: {gone})"
);
assert_eq!(registered(&github.calls()), Vec::<&String>::new());
handle.store.flush().await;
let job = handle.store.job(1).await.unwrap().unwrap();
assert_eq!(job.summary.state, JobState::Running);
}
}
#[tokio::test]
async fn runnerd_stop_tries_again_a_removal_docker_refused_and_leaves_nothing() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (mut engine, handle) = busy_and_idle(&github, &docker, &dir);
github
.0
.lock()
.unwrap()
.runners
.iter_mut()
.for_each(|runner| runner.busy = false);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
docker.0.lock().unwrap().containers[0].running = false;
docker.0.lock().unwrap().fail_removes = 1;
handle.control.shutdown(Shutdown::Graceful);
tokio::time::timeout(Duration::from_secs(30), engine.schedule())
.await
.expect("the stop ends once Docker removes them");
assert_eq!(
container_names(&docker),
BTreeSet::new(),
"both runners and the mirror are removed"
);
}
#[tokio::test]
async fn a_stop_ends_within_its_deadline_while_docker_hangs_and_keeps_the_history() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = busy_and_idle(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
tokio::time::pause();
docker.set_outage(Outage::Hung);
let asked = tokio::time::Instant::now();
handle.control.shutdown(Shutdown::Force);
tokio::time::timeout(Duration::from_secs(89), run)
.await
.expect("the stop outlasted systemd's timeout")
.unwrap();
assert!(asked.elapsed() >= STOP_DEADLINE, "it gave Docker its time");
handle.store.close();
handle.store.flush().await;
let reopened = Store::open(&dir.0.join("history.db")).unwrap();
let job = reopened.job(1).await.unwrap().unwrap();
assert_eq!(job.summary.state, JobState::Running);
}
#[tokio::test]
async fn a_second_signal_during_a_stop_kills_running_jobs() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = busy_and_idle(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github.hold();
handle.control.wake.notify_one();
github.until_waiting(1).await;
handle.control.shutdown(Shutdown::Detach);
handle.control.shutdown(Shutdown::Force);
github.release();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap();
assert_eq!(
container_names(&docker),
BTreeSet::new(),
"both runners and the mirror are removed"
);
let calls = github.calls();
assert!(
calls.contains(&"delete 7".to_owned()),
"job 1's runner is deregistered: {calls:?}"
);
}
#[tokio::test]
async fn a_second_signal_while_a_signal_removes_idle_runners_kills_running_jobs() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = busy_and_idle(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github
.until_calls(
|calls| calls.iter().any(|call| call.starts_with("runners ")),
"asked for its runners",
)
.await;
github.hold_deletes();
handle.control.shutdown(Shutdown::Detach);
github.until_waiting(1).await;
handle.control.shutdown(Shutdown::Force);
github.release_deletes();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap();
assert_eq!(
container_names(&docker),
BTreeSet::new(),
"job 1's runner and the mirror are removed too"
);
}
#[tokio::test]
async fn a_stop_leaves_a_reseed_and_a_check_of_the_caches_to_the_next_start() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let seed = Names::new("runnerd").seed("app");
{
let mut state = docker.0.lock().unwrap();
state.containers.push(Container {
id: "seed".to_owned(),
name: seed.clone(),
running: true,
labels: HashMap::from([
("runnerd.seed".to_owned(), String::new()),
("runnerd.profile".to_owned(), "app".to_owned()),
]),
});
state.volume_sizes_held.send_replace(true);
}
let (mut engine, handle) = engine_in(&github, &docker, &dir);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
assert!(engine.trim.is_some(), "the caches are being checked");
assert!(!engine.reseeds.is_empty(), "app is reseeding");
handle.control.shutdown(Shutdown::Graceful);
tokio::time::timeout(Duration::from_secs(10), engine.schedule())
.await
.expect("the stop waits for neither");
assert!(container_names(&docker).contains(&seed), "the seed stays");
let (mut engine, handle) = engine_in(&github, &docker, &dir);
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
engine.publish();
assert!(
handle.status.borrow().profiles[0].reseeding,
"the next start waits for the reseed"
);
assert!(engine.trim.is_some(), "the next start checks the caches");
}
#[tokio::test]
async fn a_reload_asked_for_while_a_signal_stops_the_daemon_is_answered_as_it_went() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (engine, handle) = engine_in(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
github
.until_calls(
|calls| calls.iter().any(|call| call.starts_with("runners ")),
"asked for its runners",
)
.await;
github.hold();
handle.control.wake.notify_one();
github.until_waiting(1).await;
handle.control.shutdown(Shutdown::Detach);
let written = Arc::new(AtomicBool::new(false));
let commit = {
let written = written.clone();
Commit::new(move || {
written.store(true, Ordering::SeqCst);
Ok(None)
})
};
let control = handle.control.clone();
let reload = tokio::spawn(async move {
control
.replace_config(
Config::parse(&CONFIG.replace("slots = 2", "slots = 1")).unwrap(),
None,
None,
commit,
RunningSlots::Drain,
Author::Owner,
)
.await
});
github.release();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap();
let answer = tokio::time::timeout(Duration::from_secs(10), reload)
.await
.expect("the reload is answered")
.unwrap();
match answer {
Ok(_) => assert!(written.load(Ordering::SeqCst), "applied, so written"),
Err(ControlError::Stopped) => {
assert!(!written.load(Ordering::SeqCst), "refused, so not written");
}
Err(other) => panic!("the reload failed: {other:?}"),
}
}
#[tokio::test]
async fn the_first_tick_after_an_install_replaces_idle_runners_of_the_old_release() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let (mut engine, _handle) = busy_and_idle(&github, &docker, &dir);
for container in &mut docker.0.lock().unwrap().containers {
container
.labels
.insert(super::super::runners::SPEC.to_owned(), "0.0.1".to_owned());
}
engine.recover().await.unwrap();
engine.tick(Shutdown::No).await;
let calls = docker.calls();
assert!(
calls.contains(&"remove container-2".to_owned()),
"the idle runner goes: {calls:?}"
);
assert!(
!calls.contains(&"remove container-1".to_owned()),
"job 1's runner keeps running: {calls:?}"
);
}
fn two_jobs_recording(
github: &FakeGithub,
docker: &FakeDocker,
dir: &TestDir,
) -> (Engine<FakeGithub, FakeDocker>, Handle) {
running_runner(github, docker, "@@job 1 1\n");
let second = identity("ci", ID_A).runner_name("app", 2, 1);
{
let mut state = github.0.lock().unwrap();
state.jobs.insert(1, details(1, "in_progress", None));
state.jobs.insert(2, details(2, "in_progress", None));
state.runners.push(runner(8, &second, true));
}
{
let mut state = docker.0.lock().unwrap();
state.containers.push(runner_container(2, 8, true));
state
.container_captures
.insert("container-2".to_owned(), "@@job 2 2\n".to_owned());
}
engine_with(github, docker, dir, &records::recording())
}
async fn until_docker_call(docker: &FakeDocker, call: &str) {
for _ in 0..500 {
if docker.calls().iter().any(|made| made == call) {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
panic!("Docker was never asked to {call}: {:?}", docker.calls());
}
#[test]
fn runnerd_stop_that_docker_turns_into_a_detach_ends_within_the_deadline_and_the_next_start_keeps_the_record()
{
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let daemon = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
daemon.block_on(async {
let (engine, handle) = two_jobs_recording(&github, &docker, &dir);
let run = tokio::spawn(engine.run());
until_state(&handle, 2, JobState::Running).await;
{
let mut state = docker.0.lock().unwrap();
state.commits_held.send_replace(true);
state
.container_logs
.insert("container-2".to_owned(), records::FAILED.to_owned());
state.containers[1].running = false;
}
github
.0
.lock()
.unwrap()
.jobs
.insert(2, details(2, "completed", Some("failure")));
handle.control.wake.notify_one();
until_docker_call(&docker, "commit container-2 runnerd-record:2").await;
tokio::time::pause();
docker.set_outage(Outage::Refused);
let asked = tokio::time::Instant::now();
handle.control.shutdown(Shutdown::Graceful);
tokio::time::timeout(STOP_DEADLINE + Duration::from_secs(10), run)
.await
.expect("the detach Docker turned the stop into has the deadline of any detach")
.unwrap();
assert!(
asked.elapsed() >= STOP_DEADLINE,
"it gave the record its time"
);
handle.store.close();
handle.store.flush().await;
});
drop(daemon);
let next = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
next.block_on(async {
{
let mut state = docker.0.lock().unwrap();
state.outage = Outage::None;
state.commits_held.send_replace(false);
}
let (mut engine, handle) = engine_with(&github, &docker, &dir, &records::recording());
engine.recover().await.unwrap();
records::until_recorded(&mut engine).await;
let note = records::note(&handle, 2).await;
assert!(
note.is_some_and(|note| note.created_at.is_some()),
"job 2's runner becomes its record at the next start"
);
engine.publish();
let runner = handle.status.borrow().profiles[0].slots[0]
.runner
.as_ref()
.map(|runner| runner.name.clone());
assert_eq!(
runner,
Some(identity("ci", ID_A).runner_name("app", 1, 1)),
"job 1's runner is adopted"
);
});
}
#[tokio::test]
async fn a_failed_job_an_idle_runner_took_while_detached_becomes_its_record_at_the_next_start() {
let (github, docker, dir) = (
FakeGithub::default(),
FakeDocker::default(),
TestDir::new("stop"),
);
let config = records::recording();
let (engine, handle) = busy_and_idle_with(&github, &docker, &dir, &config);
let run = tokio::spawn(engine.run());
until_state(&handle, 1, JobState::Running).await;
github
.until_calls(
|calls| calls.iter().any(|call| call.starts_with("runners ")),
"asked for its runners",
)
.await;
github.0.lock().unwrap().busy_deletes = 1;
handle.control.shutdown(Shutdown::Detach);
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap();
drop(handle);
github
.0
.lock()
.unwrap()
.runners
.retain(|runner| runner.id != 8);
{
let mut state = docker.0.lock().unwrap();
let idle = state
.containers
.iter_mut()
.find(|container| container.id == "container-2")
.unwrap();
idle.running = false;
state
.container_logs
.insert("container-2".to_owned(), records::FAILED.to_owned());
state
.diag
.insert("container-2".to_owned(), docker::tests::diag_archive(5));
state.fail_files = 1;
state.cut_files.push_back(512);
}
tokio::time::pause();
let (mut engine, handle) = engine_with(&github, &docker, &dir, &config);
engine.recover().await.unwrap();
let left = identity("ci", ID_A).runner_name("app", 2, 1);
assert!(
container_names(&docker).contains(&left),
"the runner Docker could not read yet stays"
);
let backoff = Duration::from_secs(600);
tokio::time::sleep(backoff).await;
engine.tick(Shutdown::No).await;
assert!(
container_names(&docker).contains(&left),
"the runner whose worker log was not read stays"
);
tokio::time::sleep(backoff).await;
records::until_recorded(&mut engine).await;
let kept: Vec<u64> = handle
.records
.borrow()
.records
.iter()
.map(|record| record.job)
.collect();
assert_eq!(kept, [5], "job 5's runner becomes its record");
let calls = docker.calls();
let commit = calls
.iter()
.position(|call| call == "commit container-2 runnerd-record:5");
let remove = calls.iter().position(|call| call == "remove container-2");
assert!(
commit.is_some() && commit < remove,
"the stopped runner is committed, then goes: {calls:?}"
);
}