use super::*;
#[cfg(unix)]
fn is_open(fd: std::os::fd::RawFd) -> bool {
nix::fcntl::fcntl(fd, nix::fcntl::FcntlArg::F_GETFD).is_ok()
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_blob_from_a_live_flock_names_open_descriptors_per_sheep() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut talkative = AppConfig::minimal("api", "./srv");
talkative.channel = true;
handle
.start(vec![
normalize(AppConfig::minimal("web", "./srv")).unwrap(),
normalize(talkative).unwrap(),
])
.await
.unwrap();
let (candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
assert_eq!(candidates.len(), 2, "both sheep must reach the gate");
assert_eq!(blob.sheep().len(), 2);
for sheep in blob.sheep() {
for fd in sheep.fds().all().into_iter().flatten() {
assert!(is_open(fd), "blob names a closed descriptor: {fd}");
}
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_snapshot_names_no_channel_for_a_sheep_that_has_none() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let mut talkative = AppConfig::minimal("api", "./srv");
talkative.channel = true;
handle
.start(vec![
normalize(AppConfig::minimal("web", "./srv")).unwrap(),
normalize(talkative).unwrap(),
])
.await
.unwrap();
let (_candidates, blob, _parked) = handle.handover_snapshot(fds).await.unwrap();
let named: Vec<(&str, bool)> = blob
.sheep()
.iter()
.map(|sheep| (sheep.name(), sheep.fds().channel.is_some()))
.collect();
assert!(
named.contains(&("web", false)),
"a sheep with no channel must carry no channel descriptor: {named:?}"
);
assert!(
named.contains(&("api", true)),
"a sheep with a live channel must carry its descriptor: {named:?}"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_pump_that_never_reports_refuses_the_snapshot_instead_of_hanging() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let snapshot = tokio::time::timeout(Duration::from_secs(3600), handle.handover_snapshot(fds))
.await
.expect("a snapshot over a wedged pump must answer rather than hang")
.unwrap();
let (candidates, _blob, _parked) = snapshot;
let borrowed: Vec<crate::handover::Candidate<'_>> = candidates
.iter()
.map(crate::handover::OwnedCandidate::as_candidate)
.collect();
assert_eq!(
crate::handover::fitness(&borrowed),
crate::handover::Fitness::Refused(crate::handover::RefusedReason::PumpUnresponsive {
sheep: "wedged".to_string(),
}),
"the gate must refuse, and must name the sheep whose pump went quiet"
);
for candidate in &candidates {
assert_eq!(
candidate.pump_unresponsive,
candidate.entry.spec.config().name == "wedged",
"exactly the wedged sheep is unresponsive, not its neighbour"
);
}
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_pump_that_missed_the_deadline_is_not_in_the_parked_set() {
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]),
);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
handle
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let (_candidates, _blob, parked) =
tokio::time::timeout(Duration::from_secs(3600), handle.handover_snapshot(fds))
.await
.expect("a snapshot over a wedged pump must answer rather than hang")
.unwrap();
parked.resume().await;
let answering = runner.spawn_index_of("answering").expect("started above");
let wedged = runner.spawn_index_of("wedged").expect("started above");
let delivered = async {
while runner.resumes(answering) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), delivered)
.await
.expect("the pump that answered was parked, and is owed a resume");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_sweep_of_six_wedged_pumps_costs_one_deadline_not_six() {
const WEDGED: usize = 6;
let names: Vec<String> = (0..WEDGED).map(|i| format!("wedged{i}")).collect();
let name_refs: Vec<&str> = names.iter().map(String::as_str).collect();
let (events, _rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(vec![ProcScript::never_exits(); WEDGED])
.with_a_pump_that_never_reports(&name_refs);
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
handle
.start(
names
.iter()
.map(|name| normalize(AppConfig::minimal(name, "./srv")).unwrap())
.collect(),
)
.await
.unwrap();
let started_at = tokio::time::Instant::now();
let (candidates, blob, _parked) =
tokio::time::timeout(Duration::from_secs(3600), handle.handover_snapshot(fds))
.await
.expect("a snapshot over six wedged pumps must answer rather than hang")
.unwrap();
let elapsed = started_at.elapsed();
assert!(
elapsed < REPORT_DEADLINE * 2,
"six wedged pumps swept concurrently should cost about one \
REPORT_DEADLINE, not six; took {elapsed:?}"
);
assert_eq!(candidates.len(), WEDGED);
for candidate in &candidates {
assert!(
candidate.pump_unresponsive,
"{} must still be reported unresponsive; racing the reports \
must not blur which pump answered",
candidate.entry.spec.config().name
);
}
assert!(
candidates.windows(2).all(|w| w[0].entry.id < w[1].entry.id),
"candidates must stay in id order across a concurrent sweep"
);
assert!(
blob.sheep().windows(2).all(|w| w[0].id() < w[1].id()),
"the blob must stay in id order too: it is a file an operator may read"
);
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn the_snapshot_carries_the_actors_counters_and_slot_state() {
let dir = tempfile::tempdir().unwrap();
let (fds, _held) = daemon_fds(&dir);
let (mut actor, _ctl_rx) = actor_with_stopping_drainee(&dir, 4242, 7);
actor.sheep.get_mut(&0).unwrap().manual = Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
});
actor.sheep.get_mut(&0).unwrap().ready_failed = true;
let (reply, rx) = oneshot::channel();
actor.handle_handover_snapshot(fds, reply);
let (candidates, blob, _parked) = rx.await.unwrap().unwrap();
assert!(
!candidates.is_empty(),
"the flock must still reach the gate at all"
);
assert_eq!(
blob.sheep()[0].manual(),
Some(PendingManual {
kind: ManualKind::Stop,
origin: CommandOrigin::Operator,
}),
"a manual stop must reach the successor, kind and origin both"
);
assert_eq!(
blob.sheep()[0].pending_delete(),
Some(false),
"nothing asked for this sheep to be deleted"
);
assert_eq!(
blob.sheep()[0].ready_failed(),
Some(true),
"an earlier reload's failed verdict must reach the successor, or the rollback that \
follows it has nothing left to replace"
);
assert!(
blob.next_id() > 0,
"a successor that reissues a live id collides"
);
assert_eq!(blob.sheep()[0].epoch(), 7, "a stale timer must stay stale");
assert_eq!(
blob.sheep()[0].fds(),
CarriedFds::none(),
"a sheep with no pump has no descriptors to carry"
);
}