use std::path::{Path, PathBuf};
use shep_core::paths::ShepPaths;
use shep_core::transport::Listener;
use super::BootError;
use super::pidfile::PidfileLock;
use crate::supervisor::SupervisorHandle;
pub(super) fn apps_for_the_roll(
flock: &[crate::handover::adopt::AdoptedSheep],
) -> Vec<shep_core::config::AppConfig> {
flock
.iter()
.filter(|sheep| sheep.carried.dog().is_none())
.map(|sheep| sheep.carried.app().clone())
.collect()
}
#[must_use]
pub(crate) fn successor_handover() -> Option<Successor> {
let path = PathBuf::from(std::env::var_os(crate::handover::HANDOVER_ENV)?);
let blob = successor_handover_at(&path)?;
Some(Successor { path, blob })
}
pub(super) fn rehydrate(carried: Successor, paths: &ShepPaths) -> Result<Rehydrated, BootError> {
let Successor { path, blob } = carried;
let counters = blob.counters();
let adopted = crate::handover::adopt::adopt(&blob)
.map_err(|source| BootError::Adopt(Box::new(source)))?;
crate::handover::adopt::discard_blob(&path);
let _ = paths;
let reloads = blob.reloads().to_vec();
Ok((
PidfileLock::from_locked(adopted.pidfile),
Listener::from_unix_listener(adopted.listener),
(adopted.sheep, counters, reloads),
))
}
#[derive(Debug)]
pub(crate) struct Successor {
pub path: PathBuf,
pub blob: crate::handover::Handover,
}
fn successor_handover_at(path: &Path) -> Option<crate::handover::Handover> {
match crate::handover::Handover::read(path) {
Ok(blob) => Some(blob),
Err(error) => {
tracing::error!(
path = %path.display(),
%error,
"this process was handed a handover blob it cannot use, and is booting as if \
it were fresh; if a flock was running, it is no longer supervised"
);
None
}
}
}
type Rehydrated = (
PidfileLock,
Listener,
(
Vec<crate::handover::adopt::AdoptedSheep>,
crate::handover::Counters,
Vec<crate::supervisor::CarriedReload>,
),
);
#[derive(Debug)]
pub(crate) struct HandoverSeam {
pub(super) supervisor: SupervisorHandle,
pub(super) fds: crate::handover::DaemonFds,
pub(super) paths: ShepPaths,
}
pub(super) async fn hand_over_now(
seam: &HandoverSeam,
) -> Result<core::convert::Infallible, String> {
let (candidates, blob, parked) = seam
.supervisor
.handover_snapshot(seam.fds)
.await
.map_err(|err| format!("the supervisor could not describe its flock: {err}"))?;
let refusal = match hand_over_carrying(&candidates, &blob, seam) {
Ok(never) => match never {},
Err(refusal) => refusal,
};
parked.resume().await;
Err(refusal)
}
fn hand_over_carrying(
candidates: &[crate::handover::OwnedCandidate],
blob: &crate::handover::Handover,
seam: &HandoverSeam,
) -> Result<core::convert::Infallible, String> {
let borrowed: Vec<crate::handover::Candidate<'_>> = candidates
.iter()
.map(crate::handover::OwnedCandidate::as_candidate)
.collect();
if let crate::handover::Fitness::Refused(reason) = crate::handover::fitness(&borrowed) {
return Err(reason.to_string());
}
crate::handover::adopt::dry_run(blob).map_err(|err| {
format!(
"a successor could not have adopted this flock, so none was started: {err}. This is \
a shep bug worth reporting: the descriptors are ones this shepherd opened itself, \
and the check that refused them is the successor's own. The flock is stopped and \
started instead, which is the reload an operator had before handovers existed"
)
})?;
crate::handover::hand_over(blob, &seam.paths).map_err(|err| err.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::boot::{BootOptions, SIGNAL_TEST_LOCK, boot, init_dirs};
use crate::fake::{ProcScript, ScriptedRunner};
use crate::testing::{SharedRunner, capture_logs, test_paths};
use shep_core::config::{AppConfig, normalize};
use shep_core::protocol::DogSource;
use std::sync::Arc;
use std::time::Duration;
#[tokio::test(start_paused = true)]
async fn a_sighup_over_a_flock_it_cannot_carry_refuses_before_it_execs() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()])
.with_a_pump_that_never_reports(&["wedged"]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
ctx.supervisor
.start(vec![
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a wedged log pump cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the gate must refuse before anything is exec'd: {refusal}"
);
assert!(
refusal.contains("wedged"),
"the refusal must name the sheep that held the flock back: {refusal}"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[tokio::test]
async fn a_sighup_over_a_blob_no_successor_could_adopt_refuses_before_it_execs() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(Vec::new()),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -2,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a blob naming no real listener cannot be adopted");
assert!(
refusal.contains("a successor could not have adopted this flock"),
"the rehearsal must be what refuses, not the `FD_CLOEXEC` sweep further on: {refusal}"
);
assert!(
refusal.contains("-1"),
"the refusal must name the descriptor it refused: {refusal}"
);
assert!(
!crate::handover::Handover::path(&paths).exists(),
"a refusal before the exec must leave no blob on disk"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_handover_starts_every_pump_reading_again() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 3])
.with_a_pump_that_never_reports(&["wedged"]),
);
let daemon = boot(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
ctx.supervisor
.start(vec![
normalize(AppConfig::minimal("quiet", "./srv")).unwrap(),
normalize(AppConfig::minimal("chatty", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
for sheep in 0..3 {
assert!(
runner.log_ctl_live(sheep),
"sheep {sheep} must have a live log pump before the report, or this case proves nothing"
);
}
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a wedged log pump cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the gate must refuse before anything is exec'd: {refusal}"
);
let answered: Vec<usize> = ["quiet", "chatty"]
.iter()
.map(|name| runner.spawn_index_of(name).expect("started above"))
.collect();
let wedged = runner.spawn_index_of("wedged").expect("started above");
let all_resumed = async {
while answered.iter().any(|sheep| runner.resumes(*sheep) == 0) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), all_resumed)
.await
.expect("every pump a refused handover reported to must be reading again");
for sheep in &answered {
assert_eq!(
runner.resumes(*sheep),
1,
"sheep {sheep} must be resumed once rather than repeatedly"
);
}
assert_eq!(answered.len(), 2, "two pumps must have answered the report");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_handover_abandoned_on_a_wedged_pump_resumes_the_pumps_that_parked() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]),
);
let supervisor = crate::supervisor::spawn_supervisor(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
events,
);
supervisor
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let seam = HandoverSeam {
supervisor: supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a pump that never reported cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the refusal must say the pump went quiet, not something else: {refusal}"
);
assert!(
refusal.contains("wedged"),
"the refusal must name the sheep whose pump went quiet: {refusal}"
);
let answering = runner.spawn_index_of("answering").expect("started above");
let wedged = runner.spawn_index_of("wedged").expect("started above");
let delivered = async {
while runner.resumes(answering) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), delivered)
.await
.expect("the pump that answered was parked, and a refusal owes it a resume");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
}
fn carried_for_the_roll(
name: &str,
dog: Option<DogSource>,
) -> crate::handover::adopt::AdoptedSheep {
let mut entry = crate::entry::ProcessEntry {
id: 1,
spec: normalize(AppConfig::minimal(name, "./srv")).unwrap(),
pending: None,
pending_reidentifies: false,
overridden: Vec::new(),
instance: 0,
status: shep_core::status::ProcStatus::Online,
pid: Some(4242),
restarts: 0,
started_at: None,
budget: crate::entry::RestartBudget::default(),
reload: crate::entry::ReloadState::None,
credentials: crate::privilege::SpawnIdentity::Resolved(None),
out_file: PathBuf::new(),
err_file: PathBuf::new(),
dog: None,
last_exit: None,
};
entry.dog = dog;
crate::handover::adopt::AdoptedSheep {
carried: crate::handover::CarriedSheep::from_entry(
&entry,
0,
crate::handover::CarriedFds::none(),
false,
None,
false,
None,
),
out_pipe: None,
err_pipe: None,
out_log: None,
err_log: None,
stdin_pipe: None,
channel: None,
}
}
#[test]
fn a_carried_dog_does_not_reach_the_muster_roll() {
let flock = vec![
carried_for_the_roll("web", None),
carried_for_the_roll("metrics", Some(DogSource::BuiltIn)),
carried_for_the_roll(
"log-rotate",
Some(DogSource::Adopted {
path: "/opt/bin/shep-log-rotate".to_string(),
}),
),
];
let names: Vec<String> = apps_for_the_roll(&flock)
.into_iter()
.map(|app| app.name)
.collect();
assert_eq!(
names,
vec!["web".to_string()],
"the roll is the operator's flock; a dog belongs to `shep.toml`"
);
}
fn write_blob(path: &Path, version: u32) {
std::fs::write(
path,
format!(
r#"{{"version":{version},"sheep":[],"listener_fd":3,"pidfile_fd":4,"next_id":0,"next_deadline":0,"next_action_stamp":0}}"#
),
)
.unwrap();
}
#[test]
fn a_blob_on_disk_makes_this_process_a_successor() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, 1);
assert!(successor_handover_at(&path).is_some());
}
#[test]
fn a_missing_blob_is_refused_out_loud_rather_than_silently() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("never-written.json");
let logs = capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(logs.contains("never-written.json"), "{logs}");
}
#[test]
fn a_blob_of_an_unknown_version_is_refused_out_loud() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, u32::MAX);
let logs = capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(logs.contains("version"), "{logs}");
}
#[test]
fn a_refused_blob_is_left_on_disk_for_an_operator_to_read() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, u32::MAX);
capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(path.exists(), "a refused blob is evidence, not litter");
}
}