use core::fmt;
use core::time::Duration;
use std::ffi::OsString;
use std::io::ErrorKind;
#[cfg(unix)]
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use shep_core::transport::Listener;
#[cfg(unix)]
use tokio::signal::unix::{SignalKind, signal};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinHandle;
use shep_core::paths::ShepPaths;
use shep_core::protocol::BusEvent;
#[cfg(unix)]
use shep_core::selector::ProcessSelector;
use crate::bus::{Bus, SharedEvent, new_bus};
use crate::cron::DEFAULT_MAX_CRON_SLEEP;
use crate::dogs::{DogSpec, spawn_dog_watch};
use crate::extras::{Extras, ExtrasReports, spawn_extras_reporter};
use crate::rpc::RpcContext;
use crate::runner::ProcessRunner;
use crate::server::RpcServer;
use crate::snapshot::{self, FlockRegistry, SnapshotError, SnapshotWriter, spawn_snapshot_writer};
use crate::supervisor::{SupervisorBuilder, SupervisorHandle};
#[cfg(unix)]
use crate::supervisor::SupervisorError;
pub const DIR_MODE: u32 = 0o700;
const EXTRAS_REPORT_CAPACITY: usize = 64;
#[cfg(windows)]
fn create_dir_at_dir_mode(dir: &Path) -> std::io::Result<()> {
std::fs::DirBuilder::new().recursive(true).create(dir)
}
#[cfg(unix)]
fn create_dir_at_dir_mode(dir: &Path) -> std::io::Result<()> {
std::fs::DirBuilder::new()
.mode(DIR_MODE)
.recursive(true)
.create(dir)
}
pub(crate) fn init_dirs(paths: &ShepPaths) -> Result<(), BootError> {
for dir in [&paths.home, &paths.logs, &paths.pids, &paths.run] {
create_dir_at_dir_mode(dir).map_err(|source| BootError::Io {
path: dir.clone(),
source,
})?;
#[cfg(unix)]
std::fs::set_permissions(dir, std::fs::Permissions::from_mode(DIR_MODE)).map_err(
|source| BootError::Io {
path: dir.clone(),
source,
},
)?;
}
Ok(())
}
#[must_use]
pub fn pidfile(paths: &ShepPaths) -> PathBuf {
paths.pids.join("shepd.pid")
}
#[cfg(test)]
#[cfg_attr(windows, allow(dead_code))]
fn write_pidfile(paths: &ShepPaths, pid: u32) -> Result<(), BootError> {
use std::io::Write;
use tempfile::NamedTempFile;
let path = pidfile(paths);
let mut tmp = NamedTempFile::new_in(&paths.pids).map_err(|source| BootError::Io {
path: paths.pids.clone(),
source,
})?;
tmp.write_all(pid.to_string().as_bytes())
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
tmp.as_file().sync_all().map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
tmp.persist(&path).map_err(|err| BootError::Io {
path,
source: err.error,
})?;
Ok(())
}
pub(crate) fn read_pidfile(paths: &ShepPaths) -> Result<Option<u32>, BootError> {
let path = pidfile(paths);
match std::fs::read_to_string(&path) {
Ok(contents) => Ok(contents.trim().parse::<u32>().ok()),
Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
Err(source) => Err(BootError::Io { path, source }),
}
}
#[derive(Debug)]
struct PidfileLock {
#[cfg(unix)]
flock: UnixLock,
#[cfg(windows)]
_handle: std::fs::File,
}
#[cfg(unix)]
#[derive(Debug)]
enum UnixLock {
Taken(nix::fcntl::Flock<std::fs::File>),
Adopted(std::fs::File),
}
#[cfg(unix)]
impl UnixLock {
fn file(&mut self) -> &mut std::fs::File {
match self {
Self::Taken(flock) => flock,
Self::Adopted(file) => file,
}
}
fn as_raw_fd(&self) -> std::os::fd::RawFd {
use std::os::fd::AsRawFd as _;
match self {
Self::Taken(flock) => flock.as_raw_fd(),
Self::Adopted(file) => file.as_raw_fd(),
}
}
}
#[cfg(windows)]
fn pidfile_lock_path(paths: &ShepPaths) -> PathBuf {
paths.pids.join("shepd.pid.lock")
}
impl PidfileLock {
#[cfg(unix)]
fn acquire(paths: &ShepPaths) -> Result<Self, BootError> {
let path = pidfile(paths);
let file = std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(false) .mode(0o600)
.open(&path)
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
match nix::fcntl::Flock::lock(file, nix::fcntl::FlockArg::LockExclusiveNonblock) {
Ok(flock) => Ok(Self {
flock: UnixLock::Taken(flock),
}),
Err((_file, nix::errno::Errno::EWOULDBLOCK)) => Err(BootError::AlreadyRunning {
pid: read_pidfile(paths)?,
}),
Err((_file, errno)) => Err(BootError::Io {
path,
source: errno.into(),
}),
}
}
#[cfg(windows)]
fn acquire(paths: &ShepPaths) -> Result<Self, BootError> {
use std::os::windows::fs::OpenOptionsExt as _;
const ERROR_SHARING_VIOLATION: i32 = 32;
let path = pidfile_lock_path(paths);
match std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(false)
.share_mode(0)
.open(&path)
{
Ok(handle) => Ok(Self { _handle: handle }),
Err(err) if err.raw_os_error() == Some(ERROR_SHARING_VIOLATION) => {
Err(BootError::AlreadyRunning {
pid: read_pidfile(paths)?,
})
}
Err(source) => Err(BootError::Io { path, source }),
}
}
#[cfg(windows)]
fn record(&mut self, paths: &ShepPaths, pid: u32) -> Result<(), BootError> {
use std::io::Write as _;
let path = pidfile(paths);
let mut file = std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&path)
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.write_all(pid.to_string().as_bytes())
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.sync_all()
.map_err(|source| BootError::Io { path, source })
}
#[cfg(unix)]
fn from_locked(file: std::fs::File) -> Self {
Self {
flock: UnixLock::Adopted(file),
}
}
#[cfg(unix)]
fn as_raw_fd(&self) -> std::os::fd::RawFd {
self.flock.as_raw_fd()
}
#[cfg(unix)]
fn record(&mut self, paths: &ShepPaths, pid: u32) -> Result<(), BootError> {
use std::io::{Seek, SeekFrom, Write};
let path = pidfile(paths);
let file = self.flock.file();
file.set_len(0).map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.seek(SeekFrom::Start(0))
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.write_all(pid.to_string().as_bytes())
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.sync_all()
.map_err(|source| BootError::Io { path, source })
}
}
#[cfg(unix)]
fn apps_for_the_roll(
flock: &[crate::handover::adopt::AdoptedSheep],
) -> Vec<shep_core::config::AppConfig> {
flock
.iter()
.filter(|sheep| sheep.carried.dog().is_none())
.map(|sheep| sheep.carried.app().clone())
.collect()
}
#[cfg(unix)]
#[must_use]
pub(crate) fn successor_handover() -> Option<Successor> {
let path = PathBuf::from(std::env::var_os(crate::handover::HANDOVER_ENV)?);
let blob = successor_handover_at(&path)?;
Some(Successor { path, blob })
}
#[cfg(unix)]
fn rehydrate(carried: Successor, paths: &ShepPaths) -> Result<Rehydrated, BootError> {
let Successor { path, blob } = carried;
let counters = blob.counters();
let adopted = crate::handover::adopt::adopt(&blob)
.map_err(|source| BootError::Adopt(source.to_string()))?;
crate::handover::adopt::discard_blob(&path);
let _ = paths;
let reloads = blob.reloads().to_vec();
Ok((
PidfileLock::from_locked(adopted.pidfile),
Listener::from_unix_listener(adopted.listener),
(adopted.sheep, counters, reloads),
))
}
#[cfg(unix)]
#[derive(Debug)]
pub(crate) struct Successor {
pub path: PathBuf,
pub blob: crate::handover::Handover,
}
#[cfg(unix)]
fn successor_handover_at(path: &Path) -> Option<crate::handover::Handover> {
match crate::handover::Handover::read(path) {
Ok(blob) => Some(blob),
Err(error) => {
tracing::error!(
path = %path.display(),
%error,
"this process was handed a handover blob it cannot use, and is booting as if \
it were fresh; if a flock was running, it is no longer supervised"
);
None
}
}
}
pub fn daemon_liveness(paths: &ShepPaths) -> Result<Shepherd, BootError> {
match PidfileLock::acquire(paths) {
Ok(lock) => {
drop(lock);
Ok(Shepherd::Absent)
}
Err(BootError::AlreadyRunning { pid: Some(pid) }) => Ok(Shepherd::Running(pid)),
Err(BootError::AlreadyRunning { pid: None }) => Ok(Shepherd::Booting),
Err(BootError::Io {
ref path,
ref source,
}) if source.kind() == ErrorKind::NotFound && path.starts_with(&paths.pids) => {
Ok(Shepherd::Absent)
}
Err(other) => Err(other),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Shepherd {
Absent,
Running(u32),
Booting,
}
#[must_use]
pub(crate) fn socket_path(paths: &ShepPaths, override_path: Option<&Path>) -> PathBuf {
match override_path {
Some(path) => path.to_path_buf(),
None => paths.socket.clone(),
}
}
#[cfg(unix)]
fn warn_if_socket_dir_is_loose(socket: &Path) {
let Some(parent) = socket.parent() else {
return;
};
let Ok(metadata) = std::fs::metadata(parent) else {
return;
};
if metadata.permissions().mode() & 0o022 != 0 {
tracing::warn!(
path = %parent.display(),
"control-socket directory is group- or world-writable; \
the 0700 guarantee only covers the default $SHEP_HOME/run"
);
}
}
#[allow(clippy::needless_return)]
pub(crate) fn bind_socket(paths: &ShepPaths, socket: &Path) -> Result<Listener, BootError> {
#[cfg(windows)]
{
const ERROR_ACCESS_DENIED: i32 = 5;
return match Listener::bind(socket) {
Ok(listener) => Ok(listener),
Err(err) if err.raw_os_error() == Some(ERROR_ACCESS_DENIED) => {
Err(BootError::AlreadyRunning {
pid: read_pidfile(paths)?,
})
}
Err(source) => Err(BootError::Io {
path: socket.to_path_buf(),
source,
}),
};
}
#[cfg(unix)]
{
const SUN_PATH_CAPACITY: usize = if cfg!(target_os = "linux") { 108 } else { 104 };
let len = socket.as_os_str().as_encoded_bytes().len();
if len >= SUN_PATH_CAPACITY {
return Err(BootError::SocketPathTooLong {
path: socket.to_path_buf(),
len,
limit: SUN_PATH_CAPACITY - 1,
});
}
warn_if_socket_dir_is_loose(socket);
match Listener::bind(socket) {
Ok(listener) => Ok(listener),
Err(err) if err.kind() == ErrorKind::AddrInUse => {
match std::os::unix::net::UnixStream::connect(socket) {
Ok(_) => Err(BootError::AlreadyRunning {
pid: read_pidfile(paths)?,
}),
Err(probe)
if matches!(
probe.kind(),
ErrorKind::ConnectionRefused | ErrorKind::NotFound
) =>
{
std::fs::remove_file(socket).map_err(|source| BootError::Io {
path: socket.to_path_buf(),
source,
})?;
Listener::bind(socket).map_err(|source| BootError::Io {
path: socket.to_path_buf(),
source,
})
}
Err(source) => Err(BootError::Io {
path: socket.to_path_buf(),
source,
}),
}
}
Err(source) => Err(BootError::Io {
path: socket.to_path_buf(),
source,
}),
}
}
}
pub const READY_FD_ENV: &str = "SHEP_READY_FD";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct DaemonReady {
pub(crate) pid: u32,
pub(crate) version: String,
}
fn write_ready(mut pipe: std::fs::File, ready: &DaemonReady) -> Result<(), BootError> {
use std::io::Write;
let mut line = serde_json::to_string(ready).expect("DaemonReady always serializes");
line.push('\n');
pipe.write_all(line.as_bytes())
.map_err(BootError::ReadyWrite)?;
Ok(())
}
#[derive(Debug, Default)]
pub struct BootOptions {
pub socket: Option<PathBuf>,
pub ready_fd: Option<std::fs::File>,
pub restore: bool,
pub max_cron_sleep: Option<Duration>,
pub notify_socket: Option<OsString>,
pub dogs: Vec<DogSpec>,
pub known_dogs: Vec<String>,
pub boot_first_dogs: Vec<String>,
pub delete_flock_on_shutdown: bool,
pub handover: bool,
}
pub async fn boot<R: ProcessRunner>(
runner: R,
paths: ShepPaths,
mut options: BootOptions,
) -> Result<RunningDaemon, BootError> {
#[cfg(unix)]
crate::handover::record_launch_path();
let delete_flock_on_shutdown = options.delete_flock_on_shutdown;
let (shutdown, shutdown_rx) = watch::channel(false);
let shutdown = Arc::new(shutdown);
#[cfg(unix)]
let (signals, connect_supervisor, connect_handover) =
install_signals(Arc::clone(&shutdown), paths.clone())?;
#[cfg(windows)]
let (signals, connect_supervisor) = install_signals(Arc::clone(&shutdown), paths.clone())?;
init_dirs(&paths)?;
let socket = socket_path(&paths, options.socket.as_deref());
#[cfg(unix)]
let (mut pidfile_lock, listener, inherited) = match successor_handover() {
Some(carried) => {
let (lock, listener, flock) = rehydrate(carried, &paths)?;
(lock, listener, Some(flock))
}
None => (
PidfileLock::acquire(&paths)?,
bind_socket(&paths, &socket)?,
None,
),
};
#[cfg(windows)]
let (mut pidfile_lock, listener) =
(PidfileLock::acquire(&paths)?, bind_socket(&paths, &socket)?);
let pid = std::process::id();
pidfile_lock.record(&paths, pid)?;
if let Some(pipe) = options.ready_fd.take() {
let ready = DaemonReady {
pid,
version: env!("CARGO_PKG_VERSION").to_string(),
};
write_ready(pipe, &ready)?;
}
let events = new_bus();
let dog_watch = spawn_dog_watch(events.subscribe(), events.clone(), paths.barks.clone());
let (breach_tx, breach_rx) = mpsc::channel(EXTRAS_REPORT_CAPACITY);
let (live_tx, live_rx) = mpsc::channel(EXTRAS_REPORT_CAPACITY);
let extras = Extras::real(
ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
},
max_cron_sleep(&options),
);
let stats = Arc::clone(&extras.stats);
let builder = SupervisorBuilder::new(runner, paths.clone(), events.clone()).extras(extras);
#[cfg(unix)]
let mut carried_apps = Vec::new();
#[cfg(unix)]
let inherited_flock = inherited.is_some();
#[cfg(unix)]
let supervisor = match inherited {
Some((flock, counters, reloads)) => {
carried_apps.extend(apps_for_the_roll(&flock));
builder
.spawn_adopted(flock, counters, reloads)
.map_err(|source| BootError::Adopt(source.to_string()))?
}
None => builder.spawn(),
};
#[cfg(windows)]
let supervisor = builder.spawn();
spawn_extras_reporter(breach_rx, live_rx, supervisor.clone());
let _ = connect_supervisor.send(supervisor.clone());
#[cfg(unix)]
let _ = connect_handover.send(options.handover.then(|| HandoverSeam {
supervisor: supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: listener.as_raw_fd(),
pidfile: pidfile_lock.as_raw_fd(),
},
paths: paths.clone(),
}));
let registry = FlockRegistry::new();
#[cfg(unix)]
for app in &carried_apps {
registry.record_config(app);
}
#[cfg(windows)]
let inherited_flock = false;
let (first, rest): (Vec<DogSpec>, Vec<DogSpec>) = options
.dogs
.iter()
.cloned()
.partition(|spec| options.boot_first_dogs.contains(&spec.name));
let dog_names: Vec<String> = options.dogs.iter().map(|dog| dog.name.clone()).collect();
crate::dogs::spawn_enabled_dogs(&first, &paths, &supervisor, &events).await;
if options.restore && !inherited_flock {
restore_flock(
&paths,
®istry,
&supervisor,
&events,
&dog_names,
&options.boot_first_dogs,
)
.await?;
}
crate::dogs::spawn_enabled_dogs(&rest, &paths, &supervisor, &events).await;
let dog_refusals = crate::dogs::DogRefusals::new();
let peer_contacts = crate::dogs::PeerContacts::new();
let silent_dog_watch = crate::dogs::spawn_silent_dog_watch(
supervisor.clone(),
dog_refusals.clone(),
peer_contacts.clone(),
events.clone(),
);
let writer = spawn_snapshot_writer(
paths.snapshot.clone(),
supervisor.clone(),
registry.clone(),
events.subscribe(),
);
let ctx = RpcContext {
supervisor,
events,
registry,
snapshot_path: paths.snapshot.clone(),
dogs_config: paths.dogs_config.clone(),
known_dogs: crate::rpc::KnownDogs::new(options.known_dogs.iter().cloned().collect()),
dog_names,
boot_first_dogs: options.boot_first_dogs.clone(),
paths: paths.clone(),
daemon_version: env!("CARGO_PKG_VERSION").to_string(),
dog_refusals,
peer_contacts,
pid,
shutdown,
stats,
};
#[cfg(unix)]
if let Some(target) = options.notify_socket.as_deref()
&& let Err(err) = crate::notify::notify(target)
{
tracing::warn!(
%err,
"readiness could not be reported to $NOTIFY_SOCKET; the flock is up regardless"
);
}
Ok(RunningDaemon {
ctx,
listener,
writer,
dog_watch,
silent_dog_watch,
paths,
socket,
shutdown_rx,
signals,
pidfile_lock,
delete_flock_on_shutdown,
})
}
fn max_cron_sleep(options: &BootOptions) -> Duration {
options.max_cron_sleep.unwrap_or(DEFAULT_MAX_CRON_SLEEP)
}
async fn restore_flock(
paths: &ShepPaths,
registry: &FlockRegistry,
supervisor: &SupervisorHandle,
events: &Bus,
dogs: &[String],
boot_first_dogs: &[String],
) -> Result<(), BootError> {
snapshot::muster(
&paths.snapshot,
registry,
supervisor,
events,
dogs,
boot_first_dogs,
)
.await?;
Ok(())
}
#[derive(Debug)]
pub struct RunningDaemon {
ctx: RpcContext,
listener: Listener,
writer: SnapshotWriter,
dog_watch: JoinHandle<()>,
silent_dog_watch: JoinHandle<()>,
paths: ShepPaths,
socket: PathBuf,
shutdown_rx: watch::Receiver<bool>,
signals: SignalTasks,
pidfile_lock: PidfileLock,
delete_flock_on_shutdown: bool,
}
impl RunningDaemon {
#[must_use]
pub fn context(&self) -> RpcContext {
self.ctx.clone()
}
#[must_use]
pub fn socket(&self) -> &Path {
&self.socket
}
pub async fn run(self) -> Result<(), BootError> {
let RunningDaemon {
ctx,
listener,
writer,
dog_watch,
silent_dog_watch,
paths,
socket,
shutdown_rx,
signals: _signals,
pidfile_lock: _pidfile_lock,
delete_flock_on_shutdown,
} = self;
RpcServer::new(listener, ctx.clone())
.serve(shutdown_rx)
.await;
writer.stop().await;
dog_watch.abort();
silent_dog_watch.abort();
let edges = ctx.registry.depends_on_by_name();
if delete_flock_on_shutdown {
ctx.registry.clear();
}
if let Err(err) = ctx.snapshot_now().await {
tracing::warn!(%err, "final muster roll write failed");
}
let _ = ctx.events.send(SharedEvent::new(BusEvent::DaemonShutdown));
crate::boot_order::stop_edges_in_reverse(edges, &ctx.dog_names, &ctx.supervisor).await;
ctx.supervisor.shutdown().await;
#[cfg(unix)]
let unlink_socket = unlink_if_present(&socket);
#[cfg(windows)]
let unlink_socket = {
let _ = &socket;
Ok(())
};
let unlink_pidfile = unlink_if_present(&pidfile(&paths));
unlink_socket.and(unlink_pidfile)
}
}
fn unlink_if_present(path: &Path) -> Result<(), BootError> {
match std::fs::remove_file(path) {
Ok(()) => Ok(()),
Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
Err(source) => Err(BootError::Io {
path: path.to_path_buf(),
source,
}),
}
}
#[derive(Debug)]
struct SignalTasks {
tasks: Vec<JoinHandle<()>>,
}
impl Drop for SignalTasks {
fn drop(&mut self) {
for task in &self.tasks {
task.abort();
}
}
}
#[cfg(windows)]
fn install_signals(
shutdown: Arc<watch::Sender<bool>>,
paths: ShepPaths,
) -> Result<(SignalTasks, oneshot::Sender<SupervisorHandle>), BootError> {
use tokio::signal::windows;
let mut signals = SignalTasks {
tasks: Vec::with_capacity(4),
};
macro_rules! listen {
($ctor:path, $name:literal) => {{
let mut stream = $ctor().map_err(|source| BootError::Io {
path: paths.home.clone(),
source,
})?;
let shutdown = Arc::clone(&shutdown);
signals.tasks.push(tokio::spawn(async move {
let mut already_shutting_down = false;
while stream.recv().await.is_some() {
if already_shutting_down {
tracing::warn!(
signal = $name,
"received a repeat shutdown event while teardown is already \
underway; teardown continues unchanged"
);
} else {
already_shutting_down = true;
}
let _ = shutdown.send(true);
}
}));
}};
}
listen!(windows::ctrl_c, "CTRL_C");
listen!(windows::ctrl_break, "CTRL_BREAK");
listen!(windows::ctrl_close, "CTRL_CLOSE");
listen!(windows::ctrl_shutdown, "CTRL_SHUTDOWN");
let (connect_supervisor, _supervisor_rx) = oneshot::channel::<SupervisorHandle>();
Ok((signals, connect_supervisor))
}
#[cfg(unix)]
fn install_signals(
shutdown: Arc<watch::Sender<bool>>,
paths: ShepPaths,
) -> Result<InstalledSignals, BootError> {
let mut signals = SignalTasks {
tasks: Vec::with_capacity(4),
};
for kind in [
SignalKind::terminate(),
SignalKind::interrupt(),
SignalKind::quit(),
] {
let mut stream = signal(kind).map_err(|source| BootError::Io {
path: paths.home.clone(),
source,
})?;
let shutdown = Arc::clone(&shutdown);
signals.tasks.push(tokio::spawn(async move {
let mut already_shutting_down = false;
while stream.recv().await.is_some() {
if already_shutting_down {
tracing::warn!(
?kind,
"received a repeat shutdown signal while teardown is already \
underway; teardown continues unchanged (SIGKILL forces an \
immediate exit)"
);
} else {
already_shutting_down = true;
}
let _ = shutdown.send(true);
}
}));
}
let mut hup = signal(SignalKind::hangup()).map_err(|source| BootError::Io {
path: paths.home.clone(),
source,
})?;
let (connect_handover, handover_rx) = oneshot::channel::<Option<HandoverSeam>>();
let hup_shutdown = Arc::clone(&shutdown);
signals.tasks.push(tokio::spawn(async move {
let seam = handover_rx.await.ok().flatten();
if hup.recv().await.is_some() {
let refusal = match &seam {
Some(seam) => match hand_over_now(seam).await {
Ok(never) => match never {},
Err(refusal) => refusal,
},
None => "this shepherd was not booted with the handover armed".to_string(),
};
tracing::warn!(
%refusal,
"SIGHUP: this flock could not be handed to a successor; stopping gracefully \
instead. This line may be the only record of the reason: a signal carries no \
sender, and the case this gate exists for is a flock that changed between a \
client's question and the signal, where that client was told nothing"
);
let _ = hup_shutdown.send(true);
}
}));
let mut usr2 = signal(SignalKind::user_defined2()).map_err(|source| BootError::Io {
path: paths.home.clone(),
source,
})?;
let (connect_supervisor, supervisor_rx) = oneshot::channel::<SupervisorHandle>();
signals.tasks.push(tokio::spawn(async move {
let Ok(supervisor) = supervisor_rx.await else {
return;
};
while usr2.recv().await.is_some() {
match supervisor.reopen(ProcessSelector::All).await {
Ok(reopened) => tracing::info!(
reopened = reopened.len(),
"SIGUSR2: every sheep's log files reopened"
),
Err(SupervisorError::NotFound) => {
tracing::info!("SIGUSR2: no sheep to reopen");
}
Err(err) => tracing::warn!(%err, "SIGUSR2: log reopen failed"),
}
}
}));
Ok((signals, connect_supervisor, connect_handover))
}
#[cfg(unix)]
type InstalledSignals = (
SignalTasks,
oneshot::Sender<SupervisorHandle>,
oneshot::Sender<Option<HandoverSeam>>,
);
#[cfg(unix)]
type Rehydrated = (
PidfileLock,
Listener,
(
Vec<crate::handover::adopt::AdoptedSheep>,
crate::handover::Counters,
Vec<crate::supervisor::CarriedReload>,
),
);
#[cfg(unix)]
#[derive(Debug)]
pub(crate) struct HandoverSeam {
supervisor: SupervisorHandle,
fds: crate::handover::DaemonFds,
paths: ShepPaths,
}
#[cfg(unix)]
async fn hand_over_now(seam: &HandoverSeam) -> Result<core::convert::Infallible, String> {
let (candidates, blob, parked) = seam
.supervisor
.handover_snapshot(seam.fds)
.await
.map_err(|err| format!("the supervisor could not describe its flock: {err}"))?;
let refusal = match hand_over_carrying(&candidates, &blob, seam) {
Ok(never) => match never {},
Err(refusal) => refusal,
};
parked.resume().await;
Err(refusal)
}
#[cfg(unix)]
fn hand_over_carrying(
candidates: &[crate::handover::OwnedCandidate],
blob: &crate::handover::Handover,
seam: &HandoverSeam,
) -> Result<core::convert::Infallible, String> {
let borrowed: Vec<crate::handover::Candidate<'_>> = candidates
.iter()
.map(crate::handover::OwnedCandidate::as_candidate)
.collect();
if let crate::handover::Fitness::Refused(reason) = crate::handover::fitness(&borrowed) {
return Err(reason.to_string());
}
crate::handover::adopt::dry_run(blob).map_err(|err| {
format!(
"a successor could not have adopted this flock, so none was started: {err}. This is \
a shep bug worth reporting: the descriptors are ones this shepherd opened itself, \
and the check that refused them is the successor's own. The flock is stopped and \
started instead, which is the reload an operator had before handovers existed"
)
})?;
crate::handover::hand_over(blob, &seam.paths).map_err(|err| err.to_string())
}
#[non_exhaustive]
#[derive(Debug)]
pub enum BootError {
Adopt(String),
Io {
path: PathBuf,
source: std::io::Error,
},
AlreadyRunning {
pid: Option<u32>,
},
Snapshot(SnapshotError),
SocketPathTooLong {
path: PathBuf,
len: usize,
limit: usize,
},
ReadyWrite(std::io::Error),
}
impl fmt::Display for BootError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io { path, source } => {
write!(f, "boot step failed for `{}`: {source}", path.display())
}
Self::AlreadyRunning { pid: Some(pid) } => {
write!(f, "a shep daemon is already running (pid {pid})")
}
Self::AlreadyRunning { pid: None } => write!(f, "a shep daemon is already running"),
Self::Snapshot(err) => write!(f, "muster roll restore failed: {err}"),
Self::SocketPathTooLong { path, len, limit } => write!(
f,
"the control socket path is {len} bytes and this platform allows {limit}: `{}`. \
A unix socket path is bounded by the kernel, not by shep, so a shorter \
$SHEP_HOME is the only fix.",
path.display()
),
Self::ReadyWrite(err) => write!(f, "writing the readiness line failed: {err}"),
Self::Adopt(reason) => write!(
f,
"this shepherd was handed a flock it could not take over: {reason}. The flock is \
still running and nothing is supervising it. `shep daemon reload` is not the \
way back: it needs a live shepherd to ask and to signal, and this process is \
about to exit without ever serving. It holds the pidfile until it does, so the \
home is claimable straight afterwards and `shep muster` starts a shepherd from \
the roll"
),
}
}
}
impl core::error::Error for BootError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Io { source, .. } => Some(source),
Self::AlreadyRunning { .. } => None,
Self::SocketPathTooLong { .. } => None,
Self::Snapshot(err) => Some(err),
Self::ReadyWrite(err) => Some(err),
Self::Adopt(_) => None,
}
}
}
impl From<SnapshotError> for BootError {
fn from(source: SnapshotError) -> Self {
Self::Snapshot(source)
}
}
#[cfg(all(test, windows))]
mod windows_tests {
use super::*;
fn paths_in(dir: &Path) -> ShepPaths {
ShepPaths::resolve(
&|key| (key == "SHEP_HOME").then(|| dir.to_string_lossy().into_owned()),
Path::new(""),
)
}
#[test]
fn init_dirs_creates_the_whole_layout() {
let dir = tempfile::tempdir().unwrap();
let paths = paths_in(dir.path());
init_dirs(&paths).unwrap();
for expected in [&paths.home, &paths.logs, &paths.pids, &paths.run] {
assert!(expected.is_dir(), "{} was not created", expected.display());
}
}
#[test]
fn a_second_pidfile_lock_is_refused_and_can_still_read_the_winners_pid() {
let dir = tempfile::tempdir().unwrap();
let paths = paths_in(dir.path());
init_dirs(&paths).unwrap();
let mut first = PidfileLock::acquire(&paths).expect("the first daemon must win");
first.record(&paths, 4242).unwrap();
let refusal = PidfileLock::acquire(&paths).expect_err("a second daemon must be refused");
let BootError::AlreadyRunning { pid } = refusal else {
panic!("a contended lock must report AlreadyRunning, got {refusal:?}");
};
assert_eq!(
pid,
Some(4242),
"the loser must be able to read the winner's pid off the pidfile"
);
}
#[test]
fn dropping_the_lock_releases_it_for_the_next_daemon() {
let dir = tempfile::tempdir().unwrap();
let paths = paths_in(dir.path());
init_dirs(&paths).unwrap();
let first = PidfileLock::acquire(&paths).unwrap();
drop(first);
PidfileLock::acquire(&paths)
.expect("a released lock must be re-acquirable, or a crash would wedge $SHEP_HOME");
}
#[tokio::test]
async fn a_second_bind_on_a_live_control_address_reports_already_running() {
let dir = tempfile::tempdir().unwrap();
let paths = paths_in(dir.path());
init_dirs(&paths).unwrap();
let _live = bind_socket(&paths, &paths.socket).expect("the first bind must succeed");
let refusal = bind_socket(&paths, &paths.socket)
.expect_err("a second daemon must not bind the same pipe");
assert!(
matches!(refusal, BootError::AlreadyRunning { .. }),
"a taken pipe name must read as AlreadyRunning, got {refusal:?}"
);
}
}
#[cfg(all(test, unix))]
mod tests {
use super::*;
use crate::dogs::DogSpec;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::snapshot::{FlockSnapshot, SNAPSHOT_VERSION, SavedApp};
use crate::testing::{AnnouncingRunner, SharedRunner, capture_logs, test_paths};
use shep_core::config::{AppConfig, ProbeConfig, ProbeKind, normalize};
use shep_core::protocol::{DogSource, ProcessEventKind};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use std::os::unix::fs::PermissionsExt;
use std::time::Duration;
fn mode_of(path: &Path) -> u32 {
std::fs::metadata(path).unwrap().permissions().mode() & 0o777
}
#[test]
fn the_adopted_pidfile_arm_does_not_release_the_lock() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let mut held = PidfileLock::acquire(&paths).expect("the predecessor must win");
let fd = std::os::fd::AsRawFd::as_raw_fd(held.flock.file());
core::mem::forget(held);
let inherited = crate::sys::adopt_handover_fd(fd)
.expect("the successor adopts the number the blob named");
let adopted = PidfileLock::from_locked(inherited);
let refusal = PidfileLock::acquire(&paths)
.expect_err("the lock must never be free while a successor holds it");
assert!(
matches!(refusal, BootError::AlreadyRunning { .. }),
"a contended lock must report AlreadyRunning, got {refusal:?}"
);
drop(adopted);
let deadline = std::time::Instant::now() + Duration::from_secs(5);
let claimed = loop {
match PidfileLock::acquire(&paths) {
Ok(claimed) => break claimed,
Err(error) => assert!(
std::time::Instant::now() < deadline,
"a successor that exits must leave the home claimable: {error:?}"
),
}
std::thread::sleep(Duration::from_millis(10));
};
drop(claimed);
}
static SIGNAL_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[test]
fn init_dirs_creates_the_whole_layout_owner_only() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
for path in [&paths.home, &paths.logs, &paths.pids, &paths.run] {
assert!(path.is_dir(), "{} was not created", path.display());
assert_eq!(mode_of(path), DIR_MODE, "{}", path.display());
}
init_dirs(&paths).unwrap(); }
#[test]
fn a_fresh_dir_lands_at_dir_mode_with_no_separate_chmod() {
let dir = tempfile::tempdir().unwrap();
let never_existed = dir.path().join("nested").join("run");
create_dir_at_dir_mode(&never_existed).unwrap();
assert_eq!(
mode_of(&never_existed),
DIR_MODE,
"a freshly created dir must be DIR_MODE at creation, not after a later chmod"
);
}
#[test]
fn init_dirs_tightens_a_world_readable_runtime_dir() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(&paths.run).unwrap();
std::fs::set_permissions(&paths.run, std::fs::Permissions::from_mode(0o755)).unwrap();
init_dirs(&paths).unwrap();
assert_eq!(
mode_of(&paths.run),
DIR_MODE,
"a loose run dir must be tightened, not accepted"
);
}
#[test]
fn pidfile_round_trips_and_reports_absence() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
assert_eq!(read_pidfile(&paths).unwrap(), None);
write_pidfile(&paths, 4242).unwrap();
assert_eq!(read_pidfile(&paths).unwrap(), Some(4242));
assert_eq!(pidfile(&paths), paths.pids.join("shepd.pid"));
}
#[test]
fn liveness_reports_none_when_no_daemon_holds_the_lock() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
assert_eq!(daemon_liveness(&paths).unwrap(), Shepherd::Absent);
}
#[test]
fn liveness_reports_none_for_a_stale_pidfile_nobody_holds() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
std::fs::write(pidfile(&paths), "999999").unwrap();
assert_eq!(daemon_liveness(&paths).unwrap(), Shepherd::Absent);
}
#[test]
fn liveness_reports_the_pid_a_lock_holder_recorded() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let mut held = PidfileLock::acquire(&paths).unwrap();
held.record(&paths, 4242).unwrap();
assert_eq!(daemon_liveness(&paths).unwrap(), Shepherd::Running(4242));
drop(held);
assert_eq!(
daemon_liveness(&paths).unwrap(),
Shepherd::Absent,
"a released lock is not a live daemon, whatever the file still says"
);
}
#[test]
fn liveness_reports_booting_for_a_holder_that_has_not_recorded_a_pid_yet() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let held = PidfileLock::acquire(&paths).unwrap();
assert_eq!(daemon_liveness(&paths).unwrap(), Shepherd::Booting);
drop(held);
assert_eq!(daemon_liveness(&paths).unwrap(), Shepherd::Absent);
}
#[test]
fn socket_path_honors_a_config_override() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
assert_eq!(socket_path(&paths, None), paths.socket);
let custom = dir.path().join("custom.sock");
assert_eq!(socket_path(&paths, Some(&custom)), custom);
}
#[tokio::test]
async fn an_over_length_socket_path_names_the_limit_and_the_variable() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let long = dir.path().join("x".repeat(200));
let err = bind_socket(&paths, &long).expect_err("a path this long cannot bind");
assert!(
matches!(err, BootError::SocketPathTooLong { .. }),
"refused before the syscall, not translated after it: {err:?}"
);
let rendered = err.to_string();
assert!(
rendered.contains("$SHEP_HOME"),
"the message names what to shorten: {rendered}"
);
assert!(
rendered.contains("bytes"),
"and the limit it is measured against: {rendered}"
);
assert!(
!rendered.contains('\u{2014}') && !rendered.contains('\u{2013}'),
"no em or en dash in copy a user reads: {rendered}"
);
}
#[tokio::test]
async fn bind_socket_binds_a_fresh_path() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let listener = bind_socket(&paths, &paths.socket).unwrap();
assert!(paths.socket.exists());
drop(listener);
}
#[track_caller]
fn stale_socket_leftover(socket: &Path) {
for _ in 0..20 {
let _ = std::fs::remove_file(socket);
drop(std::os::unix::net::UnixListener::bind(socket).unwrap());
for _ in 0..40 {
match std::os::unix::net::UnixStream::connect(socket) {
Err(refused)
if matches!(
refused.kind(),
ErrorKind::ConnectionRefused | ErrorKind::NotFound
) =>
{
return;
}
Ok(_) => std::thread::sleep(Duration::from_millis(5)),
Err(other) => {
panic!("probing the fabricated leftover socket failed: {other}")
}
}
}
}
panic!(
"{} never went stale: something kept answering on it",
socket.display()
);
}
#[tokio::test]
async fn a_socket_left_by_a_crash_is_unlinked_and_rebound() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
stale_socket_leftover(&paths.socket);
assert!(paths.socket.exists(), "the stale file must still be there");
let listener = bind_socket(&paths, &paths.socket).unwrap();
assert!(paths.socket.exists());
drop(listener);
}
#[tokio::test]
async fn a_live_socket_is_reported_as_already_running() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let live = tokio::net::UnixListener::bind(&paths.socket).unwrap();
write_pidfile(&paths, 4242).unwrap();
assert!(matches!(
bind_socket(&paths, &paths.socket),
Err(BootError::AlreadyRunning { pid: Some(4242) })
));
assert!(
paths.socket.exists(),
"a live daemon's socket must never be unlinked"
);
std::os::unix::net::UnixStream::connect(&paths.socket)
.expect("the live listener must still be accepting after a refused bind");
drop(live);
}
#[derive(Debug)]
enum RaceOutcome {
Won { socket_still_accepts: bool },
AlreadyRunning,
Other(String),
}
#[test]
fn two_concurrent_boots_on_a_stale_socket_exactly_one_wins() {
for _ in 0..25 {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
stale_socket_leftover(&paths.socket);
let barrier = Arc::new(std::sync::Barrier::new(2));
let handles: Vec<_> = (0..2)
.map(|_| {
let paths = paths.clone();
let barrier = Arc::clone(&barrier);
std::thread::spawn(move || {
barrier.wait(); let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(async {
match boot(ScriptedRunner::new(vec![]), paths, BootOptions::default())
.await
{
Ok(daemon) => {
let reachable =
std::os::unix::net::UnixStream::connect(daemon.socket())
.is_ok();
RaceOutcome::Won {
socket_still_accepts: reachable,
}
}
Err(BootError::AlreadyRunning { .. }) => {
RaceOutcome::AlreadyRunning
}
Err(other) => RaceOutcome::Other(other.to_string()),
}
})
})
})
.collect();
let outcomes: Vec<RaceOutcome> =
handles.into_iter().map(|h| h.join().unwrap()).collect();
for outcome in &outcomes {
if let RaceOutcome::Other(msg) = outcome {
panic!("a racer hit neither Ok nor AlreadyRunning: {msg}");
}
}
let wins = outcomes
.iter()
.filter(|o| matches!(o, RaceOutcome::Won { .. }))
.count();
let already_running = outcomes
.iter()
.filter(|o| matches!(o, RaceOutcome::AlreadyRunning))
.count();
assert_eq!(
wins, 1,
"exactly one racer must win a boot on the same $SHEP_HOME: {outcomes:?}"
);
assert_eq!(
already_running, 1,
"the loser must be refused as AlreadyRunning, not silently succeed or hit some other error: {outcomes:?}"
);
assert!(
matches!(
outcomes
.iter()
.find(|o| matches!(o, RaceOutcome::Won { .. }))
.unwrap(),
RaceOutcome::Won {
socket_still_accepts: true
}
),
"the winner's own socket must still accept a connection, proving its bind \
wasn't the one the loser's remove_file clobbered: {outcomes:?}"
);
}
}
#[test]
fn readiness_reports_pid_and_version_then_closes_the_pipe() {
use std::io::Read;
let (mut reader, writer) = std::io::pipe().unwrap();
let pipe = std::fs::File::from(std::os::fd::OwnedFd::from(writer));
let ready = DaemonReady {
pid: 4242,
version: "0.1.0".to_string(),
};
write_ready(pipe, &ready).unwrap();
let mut line = String::new();
reader.read_to_string(&mut line).unwrap();
assert_eq!(line.trim_end(), serde_json::to_string(&ready).unwrap());
assert!(line.ends_with('\n'), "the parent reads a line: {line:?}");
}
#[tokio::test]
async fn boot_writes_readiness_to_the_callers_pipe_after_the_socket_is_bound() {
use std::io::Read;
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (mut reader, writer) = std::io::pipe().unwrap();
let pipe = std::fs::File::from(std::os::fd::OwnedFd::from(writer));
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions {
ready_fd: Some(pipe),
..BootOptions::default()
},
)
.await
.unwrap();
assert!(
paths.socket.exists(),
"boot must bind the socket before it returns"
);
let mut line = String::new();
reader.read_to_string(&mut line).unwrap();
let ready: DaemonReady = serde_json::from_str(line.trim_end()).unwrap();
assert_eq!(ready.pid, std::process::id());
assert!(line.ends_with('\n'), "the parent reads a line: {line:?}");
drop(daemon); }
#[tokio::test]
async fn boot_restores_a_saved_flock_and_tears_down_in_order() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 99,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 1, "the muster roll must be back on its feet");
assert_eq!(flock[0].name, "web");
let run = tokio::spawn(daemon.run());
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
let final_roll = crate::snapshot::read(&paths.snapshot).unwrap();
assert_eq!(
final_roll.apps[0].instances_running, 1,
"the roll must be written before the flock is killed, or muster restores nothing"
);
assert!(
!paths.socket.exists(),
"the socket is unlinked on a clean exit"
);
assert_eq!(read_pidfile(&paths).unwrap(), None);
}
#[tokio::test]
async fn delete_flock_on_shutdown_clears_the_roll_even_on_a_signalled_exit() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
delete_flock_on_shutdown: true,
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let held = shep_core::config::normalize(AppConfig::minimal("held", "./held")).unwrap();
ctx.registry.record(std::slice::from_ref(&held));
ctx.supervisor.start(vec![held]).await.unwrap();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 1, "the held app must actually be up");
let run = tokio::spawn(daemon.run());
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
let final_roll = crate::snapshot::read(&paths.snapshot).unwrap();
assert!(
final_roll.apps.is_empty(),
"delete_flock_on_shutdown must leave the roll empty, not {:?}",
final_roll.apps
);
}
#[tokio::test]
async fn a_dev_session_still_stops_its_flock_in_dependency_order() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::ignores_signals(),
]),
paths.clone(),
BootOptions {
delete_flock_on_shutdown: true,
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let db = AppConfig::minimal("db", "./db");
let mut worker = AppConfig::minimal("worker", "./worker");
worker.depends_on = vec!["db".to_string()];
worker.kill_timeout = shep_core::values::UpDuration::from_millis(100);
let apps = shep_core::config::normalize_all(vec![db, worker]).unwrap();
ctx.registry.record(&apps);
ctx.supervisor.start(apps).await.unwrap();
let mut rx = ctx.events.subscribe();
let run = tokio::spawn(daemon.run());
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
let mut stops = Vec::new();
while let Ok(event) = rx.try_recv() {
if let shep_core::protocol::BusEvent::Process {
event: shep_core::protocol::ProcessEventKind::Stop,
info,
..
} = &*event
{
stops.push(info.name.clone());
}
}
assert_eq!(
stops,
vec!["worker".to_string(), "db".to_string()],
"the dependant stops before what it waits on"
);
}
#[tokio::test]
async fn a_dog_is_stopped_by_the_time_run_returns() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(
flock.len(),
1,
"the dog has to be up for its stop to mean anything: {flock:?}"
);
let mut rx = ctx.events.subscribe();
let run = tokio::spawn(daemon.run());
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
let mut stops = Vec::new();
while let Ok(event) = rx.try_recv() {
if let BusEvent::Process {
event: ProcessEventKind::Stop,
info,
..
} = &*event
{
stops.push(info.name.clone());
}
}
assert!(
stops.iter().any(|name| name == "metrics"),
"the dog must be stopped before run() returns; stops were {stops:?}"
);
}
#[tokio::test]
async fn boot_restores_the_flock_before_it_lets_the_dogs_out() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 1,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 2, "the sheep and the dog must both be up");
let sheep = flock
.iter()
.find(|p| p.name == "web")
.expect("the restored sheep must be present");
let dog = flock
.iter()
.find(|p| p.name == "metrics")
.expect("the dog must be present");
assert!(
sheep.dog.is_none(),
"the restored app must carry no dog marker"
);
assert_eq!(
dog.dog,
Some(DogSource::BuiltIn),
"the dog entry must carry its source"
);
assert!(
sheep.pid < dog.pid,
"the sheep must be spawned before the dog: sheep={:?} dog={:?}",
sheep.pid,
dog.pid
);
drop(daemon); }
#[test]
fn a_dog_that_will_not_start_does_not_fail_the_boot() {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let mut boot_result = None;
let logs = capture_logs(|| {
boot_result = Some(rt.block_on(boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions {
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)));
});
let daemon = boot_result
.unwrap()
.expect("a dog that will not start must not fail the boot");
let flock = rt
.block_on(daemon.context().supervisor.list_checked())
.unwrap();
let dog = flock
.iter()
.find(|p| p.name == "metrics")
.expect("the dog's entry must still be registered");
assert_eq!(
dog.status,
ProcStatus::Errored,
"a dog that could not spawn is errored, not silently absent"
);
assert!(
logs.contains("metrics"),
"the warning must name the dog that did not start: {logs:?}"
);
assert!(
logs.contains("WARN"),
"a dog failing to start is a warning, not silence: {logs:?}"
);
drop(daemon); }
#[test]
fn a_dog_enabled_under_a_sheeps_name_does_not_start_and_does_not_fail_the_boot() {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("metrics", "./srv"),
instances_running: 1,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let mut boot_result = None;
let logs = capture_logs(|| {
boot_result = Some(rt.block_on(boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)));
});
let daemon = boot_result
.unwrap()
.expect("a name collision must not fail the boot");
let flock = rt
.block_on(daemon.context().supervisor.list_checked())
.unwrap();
assert_eq!(
flock.len(),
1,
"the collision must not register a second entry: {flock:?}"
);
assert!(
flock[0].dog.is_none(),
"the sheep must not be relabeled as a dog by a same-named enable: {:?}",
flock[0]
);
assert!(
logs.contains("metrics"),
"the warning must name the collision: {logs:?}"
);
drop(daemon); }
#[test]
fn a_promoted_dog_that_takes_a_saved_sheeps_name_says_so() {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("metrics", "./srv"),
instances_running: 1,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let mut boot_result = None;
let logs = capture_logs(|| {
boot_result = Some(rt.block_on(boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
boot_first_dogs: vec!["metrics".to_string()],
..BootOptions::default()
},
)));
});
let daemon = boot_result
.unwrap()
.expect("a name collision must not fail the boot");
let flock = rt
.block_on(daemon.context().supervisor.list_checked())
.unwrap();
assert_eq!(flock.len(), 1, "one name is one entry: {flock:?}");
assert!(
flock[0].dog.is_some(),
"the promoted dog is what holds the name: {:?}",
flock[0]
);
assert!(
logs.contains("is not restored"),
"the lost sheep must be named out loud: {logs:?}"
);
drop(daemon); }
#[tokio::test]
async fn readiness_is_reported_only_once_the_roll_is_restored() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
crate::snapshot::write_atomic(
&paths.snapshot,
&FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 1,
}],
},
)
.unwrap();
let notify_path = dir.path().join("n.sock");
let listener = std::os::unix::net::UnixDatagram::bind(¬ify_path).unwrap();
listener
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let runner = AnnouncingRunner::new(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
¬ify_path,
);
let daemon = boot(
runner,
paths.clone(),
BootOptions {
restore: true,
notify_socket: Some(notify_path.clone().into_os_string()),
..BootOptions::default()
},
)
.await
.unwrap();
let mut buf = [0u8; 64];
let read = listener.recv(&mut buf).unwrap();
assert_eq!(
&buf[..read],
b"SPAWNED\n",
"READY=1 arrived before the roll was restored: a unit that goes \
green at exec time reports a flock that is not up yet, and a \
restore that hangs reads as a healthy service supervising nothing"
);
let read = listener.recv(&mut buf).unwrap();
assert_eq!(&buf[..read], b"READY=1\n");
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 1, "the roll was actually restored");
assert_eq!(flock[0].name, "web");
ctx.shutdown();
daemon.run().await.unwrap();
}
#[tokio::test]
async fn a_readiness_datagram_that_cannot_be_delivered_does_not_fail_the_boot() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions {
notify_socket: Some(dir.path().join("nobody.sock").into_os_string()),
..BootOptions::default()
},
)
.await
.expect("a daemon nobody could be told about is still a daemon");
assert!(daemon.context().supervisor.list_checked().await.is_ok());
daemon.context().shutdown();
daemon.run().await.unwrap();
}
#[tokio::test]
async fn sigterm_triggers_the_same_graceful_shutdown() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let run = tokio::spawn(daemon.run());
nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM).unwrap();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
assert!(!paths.socket.exists());
}
#[tokio::test]
async fn sighup_triggers_the_same_graceful_shutdown() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let run = tokio::spawn(daemon.run());
nix::sys::signal::raise(nix::sys::signal::Signal::SIGHUP).unwrap();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
assert!(!paths.socket.exists());
}
#[tokio::test(start_paused = true)]
async fn a_sighup_over_a_flock_it_cannot_carry_refuses_before_it_execs() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()])
.with_a_pump_that_never_reports(&["wedged"]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
ctx.supervisor
.start(vec![
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a wedged log pump cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the gate must refuse before anything is exec'd: {refusal}"
);
assert!(
refusal.contains("wedged"),
"the refusal must name the sheep that held the flock back: {refusal}"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn a_sighup_over_a_blob_no_successor_could_adopt_refuses_before_it_execs() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(Vec::new()),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -2,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a blob naming no real listener cannot be adopted");
assert!(
refusal.contains("a successor could not have adopted this flock"),
"the rehearsal must be what refuses, not the `FD_CLOEXEC` sweep further on: {refusal}"
);
assert!(
refusal.contains("-1"),
"the refusal must name the descriptor it refused: {refusal}"
);
assert!(
!crate::handover::Handover::path(&paths).exists(),
"a refusal before the exec must leave no blob on disk"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn an_abandoned_handover_starts_every_pump_reading_again() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 3])
.with_a_pump_that_never_reports(&["wedged"]),
);
let daemon = boot(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
ctx.supervisor
.start(vec![
normalize(AppConfig::minimal("quiet", "./srv")).unwrap(),
normalize(AppConfig::minimal("chatty", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
for sheep in 0..3 {
assert!(
runner.log_ctl_live(sheep),
"sheep {sheep} must have a live log pump before the report, or this case proves nothing"
);
}
let seam = HandoverSeam {
supervisor: ctx.supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a wedged log pump cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the gate must refuse before anything is exec'd: {refusal}"
);
let answered: Vec<usize> = ["quiet", "chatty"]
.iter()
.map(|name| runner.spawn_index_of(name).expect("started above"))
.collect();
let wedged = runner.spawn_index_of("wedged").expect("started above");
let all_resumed = async {
while answered.iter().any(|sheep| runner.resumes(*sheep) == 0) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), all_resumed)
.await
.expect("every pump a refused handover reported to must be reading again");
for sheep in &answered {
assert_eq!(
runner.resumes(*sheep),
1,
"sheep {sheep} must be resumed once rather than repeatedly"
);
}
assert_eq!(answered.len(), 2, "two pumps must have answered the report");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), daemon.run())
.await
.unwrap()
.unwrap();
}
#[cfg(unix)]
#[tokio::test(start_paused = true)]
async fn a_handover_abandoned_on_a_wedged_pump_resumes_the_pumps_that_parked() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let runner = Arc::new(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2])
.with_a_pump_that_never_reports(&["wedged"]),
);
let supervisor = crate::supervisor::spawn_supervisor(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
events,
);
supervisor
.start(vec![
normalize(AppConfig::minimal("answering", "./srv")).unwrap(),
normalize(AppConfig::minimal("wedged", "./srv")).unwrap(),
])
.await
.unwrap();
let seam = HandoverSeam {
supervisor: supervisor.clone(),
fds: crate::handover::DaemonFds {
listener: -1,
pidfile: -1,
},
paths: paths.clone(),
};
let refusal = hand_over_now(&seam)
.await
.expect_err("a flock with a pump that never reported cannot be carried");
assert!(
refusal.contains("did not report its descriptors in time"),
"the refusal must say the pump went quiet, not something else: {refusal}"
);
assert!(
refusal.contains("wedged"),
"the refusal must name the sheep whose pump went quiet: {refusal}"
);
let answering = runner.spawn_index_of("answering").expect("started above");
let wedged = runner.spawn_index_of("wedged").expect("started above");
let delivered = async {
while runner.resumes(answering) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), delivered)
.await
.expect("the pump that answered was parked, and a refusal owes it a resume");
assert_eq!(
runner.resumes(wedged),
0,
"a pump that never answered never parked, so nothing may resume it"
);
}
#[tokio::test]
async fn a_repeat_sigterm_is_observed_not_swallowed() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (shutdown, mut shutdown_rx) = watch::channel(false);
let shutdown = Arc::new(shutdown);
let (signals, _connect_supervisor, _connect_handover) =
install_signals(shutdown, paths).unwrap();
nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM).unwrap();
tokio::time::timeout(Duration::from_secs(5), shutdown_rx.changed())
.await
.unwrap()
.unwrap();
assert!(*shutdown_rx.borrow());
assert!(
!signals.tasks[0].is_finished(),
"the SIGTERM listener must still be polling after its first signal, not have exited"
);
nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM).unwrap();
tokio::time::timeout(Duration::from_secs(5), shutdown_rx.changed())
.await
.unwrap()
.unwrap();
assert!(
!signals.tasks[0].is_finished(),
"the SIGTERM listener must still be armed after a SECOND signal too"
);
drop(signals); }
#[test]
fn an_unset_max_cron_sleep_falls_back_to_the_daemons_own_default() {
assert_eq!(
max_cron_sleep(&BootOptions::default()),
DEFAULT_MAX_CRON_SLEEP,
"unset means the default"
);
assert_eq!(
max_cron_sleep(&BootOptions {
max_cron_sleep: Some(Duration::from_secs(300)),
..BootOptions::default()
}),
Duration::from_secs(300),
"a configured value must reach the workers unchanged"
);
}
#[tokio::test]
async fn a_booted_daemon_restarts_a_sheep_whose_liveness_probe_fails() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits(); 4]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let mut events = ctx.events.subscribe();
let run = tokio::spawn(daemon.run());
let mut app = AppConfig::minimal("web", "./srv");
app.liveness_probe = Some(ProbeConfig {
kind: ProbeKind::Tcp,
target: addr.to_string(),
interval: UpDuration::from_millis(1_000),
timeout: UpDuration::from_millis(500),
failure_threshold: 1,
});
ctx.supervisor
.start(vec![normalize(app).unwrap()])
.await
.unwrap();
let restarted = async {
loop {
match events.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
}) => return info,
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
Err(err) => panic!("the event stream closed before a restart: {err}"),
}
}
};
let info = tokio::time::timeout(Duration::from_secs(20), restarted)
.await
.expect("a failing liveness probe must restart its sheep");
assert_eq!(info.id, 0);
assert_eq!(info.restarts, 1);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap()
.unwrap();
}
#[tokio::test]
async fn sigusr2_reopens_every_sheeps_log_files() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits(); 2]));
let daemon = boot(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let run = tokio::spawn(daemon.run());
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
ctx.supervisor
.start(vec![normalize(web).unwrap()])
.await
.unwrap();
for instance in 0..2 {
assert!(
runner.log_ctl_live(instance),
"instance {instance} must have a live log pump before the signal, or this \
case proves nothing"
);
assert_eq!(
runner.reopens(instance),
0,
"instance {instance} must not have been reopened before the signal"
);
}
nix::sys::signal::raise(nix::sys::signal::Signal::SIGUSR2).unwrap();
let both_reopened = async {
while runner.reopens(0) == 0 || runner.reopens(1) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), both_reopened)
.await
.expect("SIGUSR2 must reopen every sheep's log files");
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap()
.unwrap();
}
#[cfg(unix)]
fn carried_for_the_roll(
name: &str,
dog: Option<DogSource>,
) -> crate::handover::adopt::AdoptedSheep {
let mut entry = crate::entry::ProcessEntry {
id: 1,
spec: normalize(AppConfig::minimal(name, "./srv")).unwrap(),
pending: None,
pending_reidentifies: false,
overridden: Vec::new(),
instance: 0,
status: shep_core::status::ProcStatus::Online,
pid: Some(4242),
restarts: 0,
started_at: None,
budget: crate::entry::RestartBudget::default(),
reload: crate::entry::ReloadState::None,
credentials: crate::privilege::SpawnIdentity::Resolved(None),
out_file: PathBuf::new(),
err_file: PathBuf::new(),
dog: None,
last_exit: None,
};
entry.dog = dog;
crate::handover::adopt::AdoptedSheep {
carried: crate::handover::CarriedSheep::from_entry(
&entry,
0,
crate::handover::CarriedFds::none(),
false,
None,
false,
None,
),
out_pipe: None,
err_pipe: None,
out_log: None,
err_log: None,
stdin_pipe: None,
channel: None,
}
}
#[cfg(unix)]
#[test]
fn a_carried_dog_does_not_reach_the_muster_roll() {
let flock = vec![
carried_for_the_roll("web", None),
carried_for_the_roll("metrics", Some(DogSource::BuiltIn)),
carried_for_the_roll(
"log-rotate",
Some(DogSource::Adopted {
path: "/opt/bin/shep-log-rotate".to_string(),
}),
),
];
let names: Vec<String> = apps_for_the_roll(&flock)
.into_iter()
.map(|app| app.name)
.collect();
assert_eq!(
names,
vec!["web".to_string()],
"the roll is the operator's flock; a dog belongs to `shep.toml`"
);
}
fn write_blob(path: &Path, version: u32) {
std::fs::write(
path,
format!(
r#"{{"version":{version},"sheep":[],"listener_fd":3,"pidfile_fd":4,"next_id":0,"next_deadline":0,"next_action_stamp":0}}"#
),
)
.unwrap();
}
#[test]
fn a_blob_on_disk_makes_this_process_a_successor() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, 1);
assert!(successor_handover_at(&path).is_some());
}
#[test]
fn a_missing_blob_is_refused_out_loud_rather_than_silently() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("never-written.json");
let logs = capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(logs.contains("never-written.json"), "{logs}");
}
#[test]
fn a_blob_of_an_unknown_version_is_refused_out_loud() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, u32::MAX);
let logs = capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(logs.contains("version"), "{logs}");
}
#[test]
fn a_refused_blob_is_left_on_disk_for_an_operator_to_read() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("handover.json");
write_blob(&path, u32::MAX);
capture_logs(|| assert!(successor_handover_at(&path).is_none()));
assert!(path.exists(), "a refused blob is evidence, not litter");
}
}