use std::path::{Path, PathBuf};
use shep_core::paths::ShepPaths;
use shep_core::transport::Listener;
use super::BootError;
use super::pidfile::PidfileLock;
use crate::supervisor::SupervisorHandle;
pub(super) fn apps_for_the_roll(
flock: &[crate::handover::adopt::AdoptedSheep],
) -> Vec<shep_core::config::AppConfig> {
flock
.iter()
.filter(|sheep| sheep.carried.dog().is_none())
.map(|sheep| sheep.carried.app().clone())
.collect()
}
pub(crate) fn successor_handover() -> Result<Option<Successor>, BootError> {
match crate::handover::arrive() {
Ok(Some(arrival)) => return Ok(Some(Successor::Arrived(arrival))),
Ok(None) => {}
Err(error) => return Err(BootError::Adopt(Box::new(error))),
}
let Some(path) = std::env::var_os(crate::handover::HANDOVER_ENV).map(PathBuf::from) else {
return Ok(None);
};
Ok(successor_handover_at(&path).map(|blob| Successor::Written { path, blob }))
}
pub(super) fn rehydrate(carried: Successor) -> Result<Rehydrated, BootError> {
let (blob, written) = match carried {
Successor::Arrived(arrival) => {
let blob = crate::handover::claim(arrival)
.map_err(|source| BootError::Adopt(Box::new(source)))?;
(blob, None)
}
Successor::Written { path, blob } => (blob, Some(path)),
};
let counters = blob.counters();
let adopted = crate::handover::adopt::adopt(&blob)
.map_err(|source| BootError::Adopt(Box::new(source)))?;
if let Some(path) = written {
crate::handover::adopt::discard_blob(&path);
}
let reloads = blob.reloads().to_vec();
Ok(Rehydrated {
lock: PidfileLock::from_locked(adopted.pidfile),
listener: Listener::from_unix_listener(adopted.listener),
flock: adopted.sheep,
counters,
reloads,
listeners: adopted.listeners,
})
}
#[derive(Debug)]
pub(crate) enum Successor {
Arrived(shep_handover::Arrival<crate::handover::Handover>),
Written {
path: PathBuf,
blob: crate::handover::Handover,
},
}
fn successor_handover_at(path: &Path) -> Option<crate::handover::Handover> {
match crate::handover::Handover::read(path) {
Ok(blob) => Some(blob),
Err(error) => {
tracing::error!(
path = %path.display(),
%error,
"this process was handed a handover blob it cannot use, and is booting as if \
it were fresh; if a flock was running, it is no longer supervised"
);
None
}
}
}
const DOG_LINK_PARK: core::time::Duration = core::time::Duration::from_secs(1);
pub(super) struct Rehydrated {
pub(super) lock: PidfileLock,
pub(super) listener: Listener,
pub(super) flock: Vec<crate::handover::adopt::AdoptedSheep>,
pub(super) counters: crate::handover::Counters,
pub(super) reloads: Vec<crate::supervisor::CarriedReload>,
pub(super) listeners: Vec<crate::handover::adopt::AdoptedListeners>,
}
#[derive(Debug)]
pub(crate) struct HandoverSeam {
pub(super) supervisor: SupervisorHandle,
pub(super) fds: crate::handover::DaemonFds,
pub(super) paths: ShepPaths,
pub(super) dog_links: crate::server::DogLinks,
}
pub(super) async fn hand_over_now(
seam: &HandoverSeam,
) -> Result<core::convert::Infallible, String> {
let (candidates, mut blob, parked) = seam
.supervisor
.handover_snapshot(seam.fds)
.await
.map_err(|err| format!("the supervisor could not describe its flock: {err}"))?;
let dog_links = seam.dog_links.freeze(DOG_LINK_PARK).await;
blob.carry_dog_links(&dog_links);
let refusal = match hand_over_carrying(&candidates, &blob, seam) {
Ok(never) => match never {},
Err(refusal) => refusal,
};
seam.dog_links.thaw();
parked.resume().await;
Err(refusal)
}
fn hand_over_carrying(
candidates: &[crate::handover::OwnedCandidate],
blob: &crate::handover::Handover,
seam: &HandoverSeam,
) -> Result<core::convert::Infallible, String> {
let borrowed: Vec<crate::handover::Candidate<'_>> = candidates
.iter()
.map(crate::handover::OwnedCandidate::as_candidate)
.collect();
if let crate::handover::Fitness::Refused(reason) = crate::handover::fitness(&borrowed) {
return Err(reason.to_string());
}
crate::handover::adopt::dry_run(blob).map_err(|err| {
format!(
"a successor could not have adopted this flock, so none was started: {err}. This is \
a shep bug worth reporting: the descriptors are ones this shepherd opened itself, \
and the check that refused them is the successor's own. The flock is stopped and \
started instead, which is the reload an operator had before handovers existed"
)
})?;
crate::handover::hand_over(blob, &seam.paths).map_err(|err| err.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::boot::{BootOptions, SIGNAL_TEST_LOCK, boot_with_idle_sweep, init_dirs};
use crate::fake::{ProcScript, ScriptedRunner};
use crate::testing::{SharedRunner, app_with, capture_logs, plain_entry, test_paths};
use shep_core::config::{AppConfig, normalize};
use shep_core::protocol::DogSource;
use std::sync::Arc;
use std::time::Duration;
#[tokio::test(start_paused = true)]
async fn a_sighup_over_a_flock_it_cannot_carry_refuses_before_it_execs() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot_with_idle_sweep(
ScriptedRunner::new(vec![ProcScript::never_exits()])
.with_a_pump_that_never_reports(&["wedged"]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
ctx.supervisor
.start(vec![
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
dog_links: crate::server::DogLinks::default(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a wedged log pump cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the gate must refuse before anything is exec'd: {refusal}"
);
assert!(
refusal.contains("wedged"),
"the refusal must name the sheep that held the flock back: {refusal}"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[tokio::test]
async fn a_sighup_over_a_blob_no_successor_could_adopt_refuses_before_it_execs() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot_with_idle_sweep(
ScriptedRunner::new(Vec::new()),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -2,
},
paths: paths.clone(),
dog_links: crate::server::DogLinks::default(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a blob naming no real listener cannot be adopted");
assert!(
refusal.contains("a successor could not have adopted this flock"),
"the rehearsal must be what refuses, not the exec's descriptor check further on: \
{refusal}"
);
assert!(
refusal.contains("-1"),
"the refusal must name the descriptor it refused: {refusal}"
);
assert!(
!crate::handover::Handover::path(&paths).exists(),
"a refusal before the exec must leave no blob on disk"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn an_abandoned_handover_starts_every_pump_reading_again() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 3])
.with_a_pump_that_never_reports(&["wedged"]),
);
let daemon = boot_with_idle_sweep(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
ctx.supervisor
.start(vec![
normalize(AppConfig::minimal("quiet", "./srv")).unwrap(),
normalize(AppConfig::minimal("chatty", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
for sheep in 0..3 {
assert!(
runner.log_ctl_live(sheep),
"sheep {sheep} must have a live log pump before the report, or this case proves nothing"
);
}
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
dog_links: crate::server::DogLinks::default(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a wedged log pump cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the gate must refuse before anything is exec'd: {refusal}"
);
let answered: Vec<usize> = ["quiet", "chatty"]
.iter()
.map(|name| runner.spawn_index_of(name).expect("started above"))
.collect();
let wedged = runner.spawn_index_of("wedged").expect("started above");
let all_resumed = async {
while answered.iter().any(|sheep| runner.resumes(*sheep) == 0) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), all_resumed)
.await
.expect("every pump a refused handover reported to must be reading again");
for sheep in &answered {
assert_eq!(
runner.resumes(*sheep),
1,
"sheep {sheep} must be resumed once rather than repeatedly"
);
}
assert_eq!(answered.len(), 2, "two pumps must have answered the report");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[tokio::test(start_paused = true)]
async fn a_handover_abandoned_on_a_wedged_pump_resumes_the_pumps_that_parked() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]),
);
let supervisor = crate::supervisor::spawn_supervisor(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
events,
);
supervisor
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let seam = HandoverSeam {
supervisor: supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
dog_links: crate::server::DogLinks::default(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a pump that never reported cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the refusal must say the pump went quiet, not something else: {refusal}"
);
assert!(
refusal.contains("wedged"),
"the refusal must name the sheep whose pump went quiet: {refusal}"
);
let answering = runner.spawn_index_of("answering").expect("started above");
let wedged = runner.spawn_index_of("wedged").expect("started above");
let delivered = async {
while runner.resumes(answering) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), delivered)
.await
.expect("the pump that answered was parked, and a refusal owes it a resume");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
}
fn carried_for_the_roll(
name: &str,
dog: Option<DogSource>,
) -> crate::handover::adopt::AdoptedSheep {
let entry = crate::entry::ProcessEntry {
pid: Some(4242),
dog,
..plain_entry(1, app_with(name, |_| {}))
};
crate::handover::adopt::AdoptedSheep {
carried: crate::handover::CarriedSheep::from_entry(
&entry,
0,
crate::handover::CarriedFds::none(),
false,
None,
false,
None,
),
out_pipe: None,
err_pipe: None,
out_log: None,
err_log: None,
stdin_pipe: None,
channel: None,
dog_link: None,
}
}
#[test]
fn a_carried_dog_does_not_reach_the_muster_roll() {
let flock = vec![
carried_for_the_roll("web", None),
carried_for_the_roll("metrics", Some(DogSource::BuiltIn { grants: None })),
carried_for_the_roll(
"log-rotate",
Some(DogSource::Adopted {
grants: None,
path: "/opt/bin/shep-log-rotate".to_string(),
channel: false,
}),
),
];
let names: Vec<String> = apps_for_the_roll(&flock)
.into_iter()
.map(|app| app.name)
.collect();
assert_eq!(
names,
vec!["web".to_string()],
"the roll is the operator's flock; a dog belongs to `shep.toml`"
);
}
fn write_blob(path: &Path, version: u32) {
std::fs::write(
path,
format!(
r#"{{"version":{version},"sheep":[],"listener_fd":3,"pidfile_fd":4,"next_id":0,"next_deadline":0,"next_action_stamp":0}}"#
),
)
.unwrap();
}
#[test]
fn a_blob_on_disk_makes_this_process_a_successor() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, 1);
assert!(successor_handover_at(&path).is_some());
}
#[test]
fn a_missing_blob_is_refused_out_loud_rather_than_silently() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("never-written.json");
let logs = capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(logs.contains("never-written.json"), "{logs}");
}
#[test]
fn a_blob_of_an_unknown_version_is_refused_out_loud() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, u32::MAX);
let logs = capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(logs.contains("version"), "{logs}");
}
#[test]
fn a_refused_blob_is_left_on_disk_for_an_operator_to_read() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, u32::MAX);
capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(path.exists(), "a refused blob is evidence, not litter");
}
const LEGACY_STAGE: &str = "SHEP_HANDOVER_LEGACY_STAGE";
const LEGACY_NAME: &str =
"boot::handover::tests::a_release_before_the_envelope_hands_over_to_this_one";
const LEGACY_FD: i32 = 11;
#[test]
fn a_release_before_the_envelope_hands_over_to_this_one() {
use command_fds::{CommandFdExt as _, FdMapping};
use std::io::Write as _;
if std::env::var_os(LEGACY_STAGE).is_some() {
legacy_successor_stage();
}
let dir = tempfile::tempdir().unwrap();
let blob = dir.path().join("handover.json");
std::fs::write(
&blob,
format!(
r#"{{"version":1,"sheep":[],"listener_fd":10,"pidfile_fd":{LEGACY_FD},"next_id":17,"next_deadline":3,"next_action_stamp":4}}"#
),
)
.unwrap();
let (reader, mut writer) = std::io::pipe().unwrap();
writer.write_all(b"from the old shepherd").unwrap();
drop(writer);
let out = std::process::Command::new(std::env::current_exe().unwrap())
.arg(LEGACY_NAME)
.arg("--exact")
.arg("--nocapture")
.env(LEGACY_STAGE, "1")
.env(crate::handover::HANDOVER_ENV, &blob)
.env_remove(shep_handover::HANDOVER_ENV)
.fd_mappings(vec![FdMapping {
parent_fd: reader.try_clone().unwrap().into(),
child_fd: LEGACY_FD,
}])
.unwrap()
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&out.stdout);
assert!(
out.status.success(),
"{stdout}{}",
String::from_utf8_lossy(&out.stderr)
);
assert!(
stdout.contains("running 1 test"),
"the child ran no test, so `{LEGACY_NAME}` is not where this test lives any more: \
{stdout}"
);
assert!(
stdout.contains("a legacy successor: next id 17, read: from the old shepherd"),
"{stdout}"
);
}
const REFUSED_STAGE: &str = "SHEP_HANDOVER_REFUSED_STAGE";
const REFUSED_NAME: &str =
"boot::handover::tests::a_refused_envelope_refuses_the_boot_rather_than_booting_fresh";
#[test]
fn a_refused_envelope_refuses_the_boot_rather_than_booting_fresh() {
if std::env::var_os(REFUSED_STAGE).is_some() {
let refused = successor_handover();
println!("refused: {}", matches!(refused, Err(BootError::Adopt(_))));
std::process::exit(0);
}
let dir = tempfile::tempdir().unwrap();
let out = std::process::Command::new("/bin/sh")
.arg("-c")
.arg(format!(
"{}=$$:\"$1\" exec \"$2\" \"$3\" --exact --nocapture",
shep_handover::HANDOVER_ENV
))
.arg("sh")
.arg(dir.path().join("lost.json"))
.arg(std::env::current_exe().unwrap())
.arg(REFUSED_NAME)
.env(REFUSED_STAGE, "1")
.env_remove(crate::handover::HANDOVER_ENV)
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&out.stdout);
assert!(
stdout.contains("running 1 test"),
"the child ran no test, so `{REFUSED_NAME}` is not where this test lives any more: \
{stdout}"
);
assert!(stdout.contains("refused: true"), "{stdout}");
}
fn legacy_successor_stage() -> ! {
let Ok(Some(Successor::Written { blob, .. })) = successor_handover() else {
panic!("a blob where SHEP_HANDOVER points is a legacy successor");
};
let mut buf = [0_u8; 64];
let read = nix::unistd::read(LEGACY_FD, &mut buf).expect("the carried descriptor is open");
println!(
"a legacy successor: next id {}, read: {}",
blob.counters().next_id,
String::from_utf8_lossy(&buf[..read])
);
std::process::exit(0);
}
}