use core::fmt;
use core::time::Duration;
use std::ffi::OsString;
use std::io::ErrorKind;
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use tokio::net::UnixListener;
use tokio::signal::unix::{SignalKind, signal};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::task::JoinHandle;
use shep_core::paths::ShepPaths;
use shep_core::protocol::BusEvent;
use shep_core::selector::ProcessSelector;
use crate::bus::new_bus;
use crate::cron::DEFAULT_MAX_CRON_SLEEP;
use crate::dogs::{DogSpec, spawn_dog_watch};
use crate::extras::{Extras, ExtrasReports, spawn_extras_reporter};
use crate::rpc::RpcContext;
use crate::runner::ProcessRunner;
use crate::server::RpcServer;
use crate::snapshot::{self, FlockRegistry, SnapshotError, SnapshotWriter, spawn_snapshot_writer};
use crate::supervisor::{SupervisorBuilder, SupervisorError, SupervisorHandle};
pub const DIR_MODE: u32 = 0o700;
const EXTRAS_REPORT_CAPACITY: usize = 64;
fn create_dir_at_dir_mode(dir: &Path) -> std::io::Result<()> {
std::fs::DirBuilder::new()
.mode(DIR_MODE)
.recursive(true)
.create(dir)
}
pub(crate) fn init_dirs(paths: &ShepPaths) -> Result<(), BootError> {
for dir in [&paths.home, &paths.logs, &paths.pids, &paths.run] {
create_dir_at_dir_mode(dir).map_err(|source| BootError::Io {
path: dir.clone(),
source,
})?;
std::fs::set_permissions(dir, std::fs::Permissions::from_mode(DIR_MODE)).map_err(
|source| BootError::Io {
path: dir.clone(),
source,
},
)?;
}
Ok(())
}
#[must_use]
pub fn pidfile(paths: &ShepPaths) -> PathBuf {
paths.pids.join("shepd.pid")
}
#[cfg(test)]
fn write_pidfile(paths: &ShepPaths, pid: u32) -> Result<(), BootError> {
use std::io::Write;
use tempfile::NamedTempFile;
let path = pidfile(paths);
let mut tmp = NamedTempFile::new_in(&paths.pids).map_err(|source| BootError::Io {
path: paths.pids.clone(),
source,
})?;
tmp.write_all(pid.to_string().as_bytes())
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
tmp.as_file().sync_all().map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
tmp.persist(&path).map_err(|err| BootError::Io {
path,
source: err.error,
})?;
Ok(())
}
pub(crate) fn read_pidfile(paths: &ShepPaths) -> Result<Option<u32>, BootError> {
let path = pidfile(paths);
match std::fs::read_to_string(&path) {
Ok(contents) => Ok(contents.trim().parse::<u32>().ok()),
Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
Err(source) => Err(BootError::Io { path, source }),
}
}
#[derive(Debug)]
struct PidfileLock(nix::fcntl::Flock<std::fs::File>);
impl PidfileLock {
fn acquire(paths: &ShepPaths) -> Result<Self, BootError> {
let path = pidfile(paths);
let file = std::fs::OpenOptions::new()
.create(true)
.write(true)
.truncate(false) .mode(0o600)
.open(&path)
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
match nix::fcntl::Flock::lock(file, nix::fcntl::FlockArg::LockExclusiveNonblock) {
Ok(lock) => Ok(Self(lock)),
Err((_file, nix::errno::Errno::EWOULDBLOCK)) => Err(BootError::AlreadyRunning {
pid: read_pidfile(paths)?,
}),
Err((_file, errno)) => Err(BootError::Io {
path,
source: errno.into(),
}),
}
}
fn record(&mut self, paths: &ShepPaths, pid: u32) -> Result<(), BootError> {
use std::io::{Seek, SeekFrom, Write};
let path = pidfile(paths);
let file = &mut *self.0;
file.set_len(0).map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.seek(SeekFrom::Start(0))
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.write_all(pid.to_string().as_bytes())
.map_err(|source| BootError::Io {
path: path.clone(),
source,
})?;
file.sync_all()
.map_err(|source| BootError::Io { path, source })
}
}
#[must_use]
pub(crate) fn socket_path(paths: &ShepPaths, override_path: Option<&Path>) -> PathBuf {
match override_path {
Some(path) => path.to_path_buf(),
None => paths.socket.clone(),
}
}
fn warn_if_socket_dir_is_loose(socket: &Path) {
let Some(parent) = socket.parent() else {
return;
};
let Ok(metadata) = std::fs::metadata(parent) else {
return;
};
if metadata.permissions().mode() & 0o022 != 0 {
tracing::warn!(
path = %parent.display(),
"control-socket directory is group- or world-writable; \
the 0700 guarantee only covers the default $SHEP_HOME/run"
);
}
}
pub(crate) fn bind_socket(paths: &ShepPaths, socket: &Path) -> Result<UnixListener, BootError> {
const SUN_PATH_CAPACITY: usize = if cfg!(target_os = "linux") { 108 } else { 104 };
let len = socket.as_os_str().as_encoded_bytes().len();
if len >= SUN_PATH_CAPACITY {
return Err(BootError::SocketPathTooLong {
path: socket.to_path_buf(),
len,
limit: SUN_PATH_CAPACITY - 1,
});
}
warn_if_socket_dir_is_loose(socket);
match UnixListener::bind(socket) {
Ok(listener) => Ok(listener),
Err(err) if err.kind() == ErrorKind::AddrInUse => {
match std::os::unix::net::UnixStream::connect(socket) {
Ok(_) => Err(BootError::AlreadyRunning {
pid: read_pidfile(paths)?,
}),
Err(probe)
if matches!(
probe.kind(),
ErrorKind::ConnectionRefused | ErrorKind::NotFound
) =>
{
std::fs::remove_file(socket).map_err(|source| BootError::Io {
path: socket.to_path_buf(),
source,
})?;
UnixListener::bind(socket).map_err(|source| BootError::Io {
path: socket.to_path_buf(),
source,
})
}
Err(source) => Err(BootError::Io {
path: socket.to_path_buf(),
source,
}),
}
}
Err(source) => Err(BootError::Io {
path: socket.to_path_buf(),
source,
}),
}
}
pub const READY_FD_ENV: &str = "SHEP_READY_FD";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct DaemonReady {
pub(crate) pid: u32,
pub(crate) version: String,
}
fn write_ready(mut pipe: std::fs::File, ready: &DaemonReady) -> Result<(), BootError> {
use std::io::Write;
let mut line = serde_json::to_string(ready).expect("DaemonReady always serializes");
line.push('\n');
pipe.write_all(line.as_bytes())
.map_err(BootError::ReadyWrite)?;
Ok(())
}
#[derive(Debug, Default)]
pub struct BootOptions {
pub socket: Option<PathBuf>,
pub ready_fd: Option<std::fs::File>,
pub restore: bool,
pub max_cron_sleep: Option<Duration>,
pub notify_socket: Option<OsString>,
pub dogs: Vec<DogSpec>,
pub delete_flock_on_shutdown: bool,
}
pub async fn boot<R: ProcessRunner>(
runner: R,
paths: ShepPaths,
mut options: BootOptions,
) -> Result<RunningDaemon, BootError> {
let delete_flock_on_shutdown = options.delete_flock_on_shutdown;
let (shutdown, shutdown_rx) = watch::channel(false);
let shutdown = Arc::new(shutdown);
let (signals, connect_supervisor) = install_signals(Arc::clone(&shutdown), paths.clone())?;
init_dirs(&paths)?;
let mut pidfile_lock = PidfileLock::acquire(&paths)?;
let socket = socket_path(&paths, options.socket.as_deref());
let listener = bind_socket(&paths, &socket)?;
let pid = std::process::id();
pidfile_lock.record(&paths, pid)?;
if let Some(pipe) = options.ready_fd.take() {
let ready = DaemonReady {
pid,
version: env!("CARGO_PKG_VERSION").to_string(),
};
write_ready(pipe, &ready)?;
}
let events = new_bus();
let dog_watch = spawn_dog_watch(events.subscribe(), paths.barks.clone());
let (breach_tx, breach_rx) = mpsc::channel(EXTRAS_REPORT_CAPACITY);
let (live_tx, live_rx) = mpsc::channel(EXTRAS_REPORT_CAPACITY);
let extras = Extras::real(
ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
},
max_cron_sleep(&options),
);
let stats = Arc::clone(&extras.stats);
let supervisor = SupervisorBuilder::new(runner, paths.clone(), events.clone())
.extras(extras)
.spawn();
spawn_extras_reporter(breach_rx, live_rx, supervisor.clone());
let _ = connect_supervisor.send(supervisor.clone());
let registry = FlockRegistry::new();
if options.restore {
restore_flock(&paths, ®istry, &supervisor).await?;
}
crate::dogs::spawn_enabled_dogs(&options.dogs, &paths, &supervisor).await;
let writer = spawn_snapshot_writer(
paths.snapshot.clone(),
supervisor.clone(),
registry.clone(),
events.subscribe(),
);
let ctx = RpcContext {
supervisor,
events,
registry,
snapshot_path: paths.snapshot.clone(),
daemon_config: paths.daemon_config.clone(),
paths: paths.clone(),
daemon_version: env!("CARGO_PKG_VERSION").to_string(),
pid,
shutdown,
stats,
};
if let Some(target) = options.notify_socket.as_deref()
&& let Err(err) = crate::notify::notify(target)
{
tracing::warn!(
%err,
"readiness could not be reported to $NOTIFY_SOCKET; the flock is up regardless"
);
}
Ok(RunningDaemon {
ctx,
listener,
writer,
dog_watch,
paths,
socket,
shutdown_rx,
signals,
pidfile_lock,
delete_flock_on_shutdown,
})
}
fn max_cron_sleep(options: &BootOptions) -> Duration {
options.max_cron_sleep.unwrap_or(DEFAULT_MAX_CRON_SLEEP)
}
async fn restore_flock(
paths: &ShepPaths,
registry: &FlockRegistry,
supervisor: &SupervisorHandle,
) -> Result<(), BootError> {
snapshot::muster(&paths.snapshot, registry, supervisor).await?;
Ok(())
}
#[derive(Debug)]
pub struct RunningDaemon {
ctx: RpcContext,
listener: UnixListener,
writer: SnapshotWriter,
dog_watch: JoinHandle<()>,
paths: ShepPaths,
socket: PathBuf,
shutdown_rx: watch::Receiver<bool>,
signals: SignalTasks,
pidfile_lock: PidfileLock,
delete_flock_on_shutdown: bool,
}
impl RunningDaemon {
#[must_use]
pub fn context(&self) -> RpcContext {
self.ctx.clone()
}
#[must_use]
pub fn socket(&self) -> &Path {
&self.socket
}
pub async fn run(self) -> Result<(), BootError> {
let RunningDaemon {
ctx,
listener,
writer,
dog_watch,
paths,
socket,
shutdown_rx,
signals: _signals,
pidfile_lock: _pidfile_lock,
delete_flock_on_shutdown,
} = self;
RpcServer::new(listener, ctx.clone())
.serve(shutdown_rx)
.await;
writer.stop().await;
dog_watch.abort();
if delete_flock_on_shutdown {
ctx.registry.clear();
}
if let Err(err) = ctx.snapshot_now().await {
tracing::warn!(%err, "final muster roll write failed");
}
let _ = ctx.events.send(BusEvent::DaemonShutdown);
ctx.supervisor.shutdown().await;
let unlink_socket = unlink_if_present(&socket);
let unlink_pidfile = unlink_if_present(&pidfile(&paths));
unlink_socket.and(unlink_pidfile)
}
}
fn unlink_if_present(path: &Path) -> Result<(), BootError> {
match std::fs::remove_file(path) {
Ok(()) => Ok(()),
Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
Err(source) => Err(BootError::Io {
path: path.to_path_buf(),
source,
}),
}
}
#[derive(Debug)]
struct SignalTasks {
tasks: Vec<JoinHandle<()>>,
}
impl Drop for SignalTasks {
fn drop(&mut self) {
for task in &self.tasks {
task.abort();
}
}
}
fn install_signals(
shutdown: Arc<watch::Sender<bool>>,
paths: ShepPaths,
) -> Result<(SignalTasks, oneshot::Sender<SupervisorHandle>), BootError> {
let mut signals = SignalTasks {
tasks: Vec::with_capacity(4),
};
for kind in [
SignalKind::terminate(),
SignalKind::interrupt(),
SignalKind::quit(),
] {
let mut stream = signal(kind).map_err(|source| BootError::Io {
path: paths.home.clone(),
source,
})?;
let shutdown = Arc::clone(&shutdown);
signals.tasks.push(tokio::spawn(async move {
let mut already_shutting_down = false;
while stream.recv().await.is_some() {
if already_shutting_down {
tracing::warn!(
?kind,
"received a repeat shutdown signal while teardown is already \
underway; teardown continues unchanged (SIGKILL forces an \
immediate exit)"
);
} else {
already_shutting_down = true;
}
let _ = shutdown.send(true);
}
}));
}
let mut usr2 = signal(SignalKind::user_defined2()).map_err(|source| BootError::Io {
path: paths.home.clone(),
source,
})?;
let (connect_supervisor, supervisor_rx) = oneshot::channel::<SupervisorHandle>();
signals.tasks.push(tokio::spawn(async move {
let Ok(supervisor) = supervisor_rx.await else {
return;
};
while usr2.recv().await.is_some() {
match supervisor.reopen(ProcessSelector::All).await {
Ok(reopened) => tracing::info!(
reopened = reopened.len(),
"SIGUSR2: every sheep's log files reopened"
),
Err(SupervisorError::NotFound) => {
tracing::info!("SIGUSR2: no sheep to reopen");
}
Err(err) => tracing::warn!(%err, "SIGUSR2: log reopen failed"),
}
}
}));
Ok((signals, connect_supervisor))
}
#[non_exhaustive]
#[derive(Debug)]
pub enum BootError {
Io {
path: PathBuf,
source: std::io::Error,
},
AlreadyRunning {
pid: Option<u32>,
},
Snapshot(SnapshotError),
SocketPathTooLong {
path: PathBuf,
len: usize,
limit: usize,
},
ReadyWrite(std::io::Error),
}
impl fmt::Display for BootError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io { path, source } => {
write!(f, "boot step failed for `{}`: {source}", path.display())
}
Self::AlreadyRunning { pid: Some(pid) } => {
write!(f, "a shep daemon is already running (pid {pid})")
}
Self::AlreadyRunning { pid: None } => write!(f, "a shep daemon is already running"),
Self::Snapshot(err) => write!(f, "muster roll restore failed: {err}"),
Self::SocketPathTooLong { path, len, limit } => write!(
f,
"the control socket path is {len} bytes and this platform allows {limit}: `{}`. \
A unix socket path is bounded by the kernel, not by shep, so a shorter \
$SHEP_HOME is the only fix.",
path.display()
),
Self::ReadyWrite(err) => write!(f, "writing the readiness line failed: {err}"),
}
}
}
impl core::error::Error for BootError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Io { source, .. } => Some(source),
Self::AlreadyRunning { .. } => None,
Self::SocketPathTooLong { .. } => None,
Self::Snapshot(err) => Some(err),
Self::ReadyWrite(err) => Some(err),
}
}
}
impl From<SnapshotError> for BootError {
fn from(source: SnapshotError) -> Self {
Self::Snapshot(source)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::dogs::DogSpec;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::snapshot::{FlockSnapshot, SNAPSHOT_VERSION, SavedApp};
use crate::testing::{AnnouncingRunner, SharedRunner, capture_logs, test_paths};
use shep_core::config::{AppConfig, ProbeConfig, ProbeKind, normalize};
use shep_core::protocol::{DogSource, ProcessEventKind};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use std::os::unix::fs::PermissionsExt;
use std::time::Duration;
fn mode_of(path: &Path) -> u32 {
std::fs::metadata(path).unwrap().permissions().mode() & 0o777
}
static SIGNAL_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[test]
fn init_dirs_creates_the_whole_layout_owner_only() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
for path in [&paths.home, &paths.logs, &paths.pids, &paths.run] {
assert!(path.is_dir(), "{} was not created", path.display());
assert_eq!(mode_of(path), DIR_MODE, "{}", path.display());
}
init_dirs(&paths).unwrap(); }
#[test]
fn a_fresh_dir_lands_at_dir_mode_with_no_separate_chmod() {
let dir = tempfile::tempdir().unwrap();
let never_existed = dir.path().join("nested").join("run");
create_dir_at_dir_mode(&never_existed).unwrap();
assert_eq!(
mode_of(&never_existed),
DIR_MODE,
"a freshly created dir must be DIR_MODE at creation, not after a later chmod"
);
}
#[test]
fn init_dirs_tightens_a_world_readable_runtime_dir() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(&paths.run).unwrap();
std::fs::set_permissions(&paths.run, std::fs::Permissions::from_mode(0o755)).unwrap();
init_dirs(&paths).unwrap();
assert_eq!(
mode_of(&paths.run),
DIR_MODE,
"a loose run dir must be tightened, not accepted"
);
}
#[test]
fn pidfile_round_trips_and_reports_absence() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
assert_eq!(read_pidfile(&paths).unwrap(), None);
write_pidfile(&paths, 4242).unwrap();
assert_eq!(read_pidfile(&paths).unwrap(), Some(4242));
assert_eq!(pidfile(&paths), paths.pids.join("shepd.pid"));
}
#[test]
fn socket_path_honors_a_config_override() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
assert_eq!(socket_path(&paths, None), paths.socket);
let custom = dir.path().join("custom.sock");
assert_eq!(socket_path(&paths, Some(&custom)), custom);
}
#[tokio::test]
async fn an_over_length_socket_path_names_the_limit_and_the_variable() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let long = dir.path().join("x".repeat(200));
let err = bind_socket(&paths, &long).expect_err("a path this long cannot bind");
assert!(
matches!(err, BootError::SocketPathTooLong { .. }),
"refused before the syscall, not translated after it: {err:?}"
);
let rendered = err.to_string();
assert!(
rendered.contains("$SHEP_HOME"),
"the message names what to shorten: {rendered}"
);
assert!(
rendered.contains("bytes"),
"and the limit it is measured against: {rendered}"
);
assert!(
!rendered.contains('\u{2014}') && !rendered.contains('\u{2013}'),
"no em or en dash in copy a user reads: {rendered}"
);
}
#[tokio::test]
async fn bind_socket_binds_a_fresh_path() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let listener = bind_socket(&paths, &paths.socket).unwrap();
assert!(paths.socket.exists());
drop(listener);
}
#[track_caller]
fn stale_socket_leftover(socket: &Path) {
for _ in 0..20 {
let _ = std::fs::remove_file(socket);
drop(std::os::unix::net::UnixListener::bind(socket).unwrap());
for _ in 0..40 {
match std::os::unix::net::UnixStream::connect(socket) {
Err(refused)
if matches!(
refused.kind(),
ErrorKind::ConnectionRefused | ErrorKind::NotFound
) =>
{
return;
}
Ok(_) => std::thread::sleep(Duration::from_millis(5)),
Err(other) => {
panic!("probing the fabricated leftover socket failed: {other}")
}
}
}
}
panic!(
"{} never went stale: something kept answering on it",
socket.display()
);
}
#[tokio::test]
async fn a_socket_left_by_a_crash_is_unlinked_and_rebound() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
stale_socket_leftover(&paths.socket);
assert!(paths.socket.exists(), "the stale file must still be there");
let listener = bind_socket(&paths, &paths.socket).unwrap();
assert!(paths.socket.exists());
drop(listener);
}
#[tokio::test]
async fn a_live_socket_is_reported_as_already_running() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let live = UnixListener::bind(&paths.socket).unwrap();
write_pidfile(&paths, 4242).unwrap();
assert!(matches!(
bind_socket(&paths, &paths.socket),
Err(BootError::AlreadyRunning { pid: Some(4242) })
));
assert!(
paths.socket.exists(),
"a live daemon's socket must never be unlinked"
);
std::os::unix::net::UnixStream::connect(&paths.socket)
.expect("the live listener must still be accepting after a refused bind");
drop(live);
}
#[derive(Debug)]
enum RaceOutcome {
Won { socket_still_accepts: bool },
AlreadyRunning,
Other(String),
}
#[test]
fn two_concurrent_boots_on_a_stale_socket_exactly_one_wins() {
for _ in 0..25 {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
stale_socket_leftover(&paths.socket);
let barrier = Arc::new(std::sync::Barrier::new(2));
let handles: Vec<_> = (0..2)
.map(|_| {
let paths = paths.clone();
let barrier = Arc::clone(&barrier);
std::thread::spawn(move || {
barrier.wait(); let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(async {
match boot(ScriptedRunner::new(vec![]), paths, BootOptions::default())
.await
{
Ok(daemon) => {
let reachable =
std::os::unix::net::UnixStream::connect(daemon.socket())
.is_ok();
RaceOutcome::Won {
socket_still_accepts: reachable,
}
}
Err(BootError::AlreadyRunning { .. }) => {
RaceOutcome::AlreadyRunning
}
Err(other) => RaceOutcome::Other(other.to_string()),
}
})
})
})
.collect();
let outcomes: Vec<RaceOutcome> =
handles.into_iter().map(|h| h.join().unwrap()).collect();
for outcome in &outcomes {
if let RaceOutcome::Other(msg) = outcome {
panic!("a racer hit neither Ok nor AlreadyRunning: {msg}");
}
}
let wins = outcomes
.iter()
.filter(|o| matches!(o, RaceOutcome::Won { .. }))
.count();
let already_running = outcomes
.iter()
.filter(|o| matches!(o, RaceOutcome::AlreadyRunning))
.count();
assert_eq!(
wins, 1,
"exactly one racer must win a boot on the same $SHEP_HOME: {outcomes:?}"
);
assert_eq!(
already_running, 1,
"the loser must be refused as AlreadyRunning, not silently succeed or hit some other error: {outcomes:?}"
);
assert!(
matches!(
outcomes
.iter()
.find(|o| matches!(o, RaceOutcome::Won { .. }))
.unwrap(),
RaceOutcome::Won {
socket_still_accepts: true
}
),
"the winner's own socket must still accept a connection, proving its bind \
wasn't the one the loser's remove_file clobbered: {outcomes:?}"
);
}
}
#[test]
fn readiness_reports_pid_and_version_then_closes_the_pipe() {
use std::io::Read;
let (mut reader, writer) = std::io::pipe().unwrap();
let pipe = std::fs::File::from(std::os::fd::OwnedFd::from(writer));
let ready = DaemonReady {
pid: 4242,
version: "0.1.0".to_string(),
};
write_ready(pipe, &ready).unwrap();
let mut line = String::new();
reader.read_to_string(&mut line).unwrap();
assert_eq!(line.trim_end(), serde_json::to_string(&ready).unwrap());
assert!(line.ends_with('\n'), "the parent reads a line: {line:?}");
}
#[tokio::test]
async fn boot_writes_readiness_to_the_callers_pipe_after_the_socket_is_bound() {
use std::io::Read;
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (mut reader, writer) = std::io::pipe().unwrap();
let pipe = std::fs::File::from(std::os::fd::OwnedFd::from(writer));
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions {
ready_fd: Some(pipe),
..BootOptions::default()
},
)
.await
.unwrap();
assert!(
paths.socket.exists(),
"boot must bind the socket before it returns"
);
let mut line = String::new();
reader.read_to_string(&mut line).unwrap();
let ready: DaemonReady = serde_json::from_str(line.trim_end()).unwrap();
assert_eq!(ready.pid, std::process::id());
assert!(line.ends_with('\n'), "the parent reads a line: {line:?}");
drop(daemon); }
#[tokio::test]
async fn boot_restores_a_saved_flock_and_tears_down_in_order() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 99,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 1, "the muster roll must be back on its feet");
assert_eq!(flock[0].name, "web");
let run = tokio::spawn(daemon.run());
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
let final_roll = crate::snapshot::read(&paths.snapshot).unwrap();
assert_eq!(
final_roll.apps[0].instances_running, 1,
"the roll must be written before the flock is killed, or muster restores nothing"
);
assert!(
!paths.socket.exists(),
"the socket is unlinked on a clean exit"
);
assert_eq!(read_pidfile(&paths).unwrap(), None);
}
#[tokio::test]
async fn delete_flock_on_shutdown_clears_the_roll_even_on_a_signalled_exit() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
delete_flock_on_shutdown: true,
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let held = shep_core::config::normalize(AppConfig::minimal("held", "./held")).unwrap();
ctx.registry.record(std::slice::from_ref(&held));
ctx.supervisor.start(vec![held]).await.unwrap();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 1, "the held app must actually be up");
let run = tokio::spawn(daemon.run());
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
let final_roll = crate::snapshot::read(&paths.snapshot).unwrap();
assert!(
final_roll.apps.is_empty(),
"delete_flock_on_shutdown must leave the roll empty, not {:?}",
final_roll.apps
);
}
#[tokio::test]
async fn boot_restores_the_flock_before_it_lets_the_dogs_out() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 1,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits(), ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)
.await
.unwrap();
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 2, "the sheep and the dog must both be up");
let sheep = flock
.iter()
.find(|p| p.name == "web")
.expect("the restored sheep must be present");
let dog = flock
.iter()
.find(|p| p.name == "metrics")
.expect("the dog must be present");
assert!(
sheep.dog.is_none(),
"the restored app must carry no dog marker"
);
assert_eq!(
dog.dog,
Some(DogSource::BuiltIn),
"the dog entry must carry its source"
);
assert!(
sheep.pid < dog.pid,
"the sheep must be spawned before the dog: sheep={:?} dog={:?}",
sheep.pid,
dog.pid
);
drop(daemon); }
#[test]
fn a_dog_that_will_not_start_does_not_fail_the_boot() {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let mut boot_result = None;
let logs = capture_logs(|| {
boot_result = Some(rt.block_on(boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions {
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)));
});
let daemon = boot_result
.unwrap()
.expect("a dog that will not start must not fail the boot");
let flock = rt
.block_on(daemon.context().supervisor.list_checked())
.unwrap();
let dog = flock
.iter()
.find(|p| p.name == "metrics")
.expect("the dog's entry must still be registered");
assert_eq!(
dog.status,
ProcStatus::Errored,
"a dog that could not spawn is errored, not silently absent"
);
assert!(
logs.contains("metrics"),
"the warning must name the dog that did not start: {logs:?}"
);
assert!(
logs.contains("WARN"),
"a dog failing to start is a warning, not silence: {logs:?}"
);
drop(daemon); }
#[test]
fn a_dog_enabled_under_a_sheeps_name_does_not_start_and_does_not_fail_the_boot() {
let _guard = SIGNAL_TEST_LOCK.blocking_lock();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("metrics", "./srv"),
instances_running: 1,
}],
};
crate::snapshot::write_atomic(&paths.snapshot, &roll).unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let mut boot_result = None;
let logs = capture_logs(|| {
boot_result = Some(rt.block_on(boot(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
BootOptions {
restore: true,
dogs: vec![DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
}],
..BootOptions::default()
},
)));
});
let daemon = boot_result
.unwrap()
.expect("a name collision must not fail the boot");
let flock = rt
.block_on(daemon.context().supervisor.list_checked())
.unwrap();
assert_eq!(
flock.len(),
1,
"the collision must not register a second entry: {flock:?}"
);
assert!(
flock[0].dog.is_none(),
"the sheep must not be relabeled as a dog by a same-named enable: {:?}",
flock[0]
);
assert!(
logs.contains("metrics"),
"the warning must name the collision: {logs:?}"
);
drop(daemon); }
#[tokio::test]
async fn readiness_is_reported_only_once_the_roll_is_restored() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
crate::snapshot::write_atomic(
&paths.snapshot,
&FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 1,
}],
},
)
.unwrap();
let notify_path = dir.path().join("n.sock");
let listener = std::os::unix::net::UnixDatagram::bind(¬ify_path).unwrap();
listener
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let runner = AnnouncingRunner::new(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
¬ify_path,
);
let daemon = boot(
runner,
paths.clone(),
BootOptions {
restore: true,
notify_socket: Some(notify_path.clone().into_os_string()),
..BootOptions::default()
},
)
.await
.unwrap();
let mut buf = [0u8; 64];
let read = listener.recv(&mut buf).unwrap();
assert_eq!(
&buf[..read],
b"SPAWNED\n",
"READY=1 arrived before the roll was restored: a unit that goes \
green at exec time reports a flock that is not up yet, and a \
restore that hangs reads as a healthy service supervising nothing"
);
let read = listener.recv(&mut buf).unwrap();
assert_eq!(&buf[..read], b"READY=1\n");
let ctx = daemon.context();
let flock = ctx.supervisor.list_checked().await.unwrap();
assert_eq!(flock.len(), 1, "the roll was actually restored");
assert_eq!(flock[0].name, "web");
ctx.shutdown();
daemon.run().await.unwrap();
}
#[tokio::test]
async fn a_readiness_datagram_that_cannot_be_delivered_does_not_fail_the_boot() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions {
notify_socket: Some(dir.path().join("nobody.sock").into_os_string()),
..BootOptions::default()
},
)
.await
.expect("a daemon nobody could be told about is still a daemon");
assert!(daemon.context().supervisor.list_checked().await.is_ok());
daemon.context().shutdown();
daemon.run().await.unwrap();
}
#[tokio::test]
async fn sigterm_triggers_the_same_graceful_shutdown() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
init_dirs(&paths).unwrap();
let daemon = boot(
ScriptedRunner::new(vec![]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let run = tokio::spawn(daemon.run());
nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM).unwrap();
tokio::time::timeout(Duration::from_secs(5), run)
.await
.unwrap()
.unwrap()
.unwrap();
assert!(!paths.socket.exists());
}
#[tokio::test]
async fn a_repeat_sigterm_is_observed_not_swallowed() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (shutdown, mut shutdown_rx) = watch::channel(false);
let shutdown = Arc::new(shutdown);
let (signals, _connect_supervisor) = install_signals(shutdown, paths).unwrap();
nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM).unwrap();
tokio::time::timeout(Duration::from_secs(5), shutdown_rx.changed())
.await
.unwrap()
.unwrap();
assert!(*shutdown_rx.borrow());
assert!(
!signals.tasks[0].is_finished(),
"the SIGTERM listener must still be polling after its first signal, not have exited"
);
nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM).unwrap();
tokio::time::timeout(Duration::from_secs(5), shutdown_rx.changed())
.await
.unwrap()
.unwrap();
assert!(
!signals.tasks[0].is_finished(),
"the SIGTERM listener must still be armed after a SECOND signal too"
);
drop(signals); }
#[test]
fn an_unset_max_cron_sleep_falls_back_to_the_daemons_own_default() {
assert_eq!(
max_cron_sleep(&BootOptions::default()),
DEFAULT_MAX_CRON_SLEEP,
"unset means the default"
);
assert_eq!(
max_cron_sleep(&BootOptions {
max_cron_sleep: Some(Duration::from_secs(300)),
..BootOptions::default()
}),
Duration::from_secs(300),
"a configured value must reach the workers unchanged"
);
}
#[tokio::test]
async fn a_booted_daemon_restarts_a_sheep_whose_liveness_probe_fails() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let reserved = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = reserved.local_addr().unwrap();
drop(reserved);
let daemon = boot(
ScriptedRunner::new(vec![ProcScript::never_exits(); 4]),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let mut events = ctx.events.subscribe();
let run = tokio::spawn(daemon.run());
let mut app = AppConfig::minimal("web", "./srv");
app.liveness_probe = Some(ProbeConfig {
kind: ProbeKind::Tcp,
target: addr.to_string(),
interval: UpDuration::from_millis(1_000),
timeout: UpDuration::from_millis(500),
failure_threshold: 1,
});
ctx.supervisor
.start(vec![normalize(app).unwrap()])
.await
.unwrap();
let restarted = async {
loop {
match events.recv().await {
Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
}) => return info,
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
Err(err) => panic!("the event stream closed before a restart: {err}"),
}
}
};
let info = tokio::time::timeout(Duration::from_secs(20), restarted)
.await
.expect("a failing liveness probe must restart its sheep");
assert_eq!(info.id, 0);
assert_eq!(info.restarts, 1);
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap()
.unwrap();
}
#[tokio::test]
async fn sigusr2_reopens_every_sheeps_log_files() {
let _guard = SIGNAL_TEST_LOCK.lock().await;
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits(); 2]));
let daemon = boot(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
BootOptions::default(),
)
.await
.unwrap();
let ctx = daemon.context();
let run = tokio::spawn(daemon.run());
let mut web = AppConfig::minimal("web", "./srv");
web.instances = 2;
ctx.supervisor
.start(vec![normalize(web).unwrap()])
.await
.unwrap();
for instance in 0..2 {
assert!(
runner.log_ctl_live(instance),
"instance {instance} must have a live log pump before the signal, or this \
case proves nothing"
);
assert_eq!(
runner.reopens(instance),
0,
"instance {instance} must not have been reopened before the signal"
);
}
nix::sys::signal::raise(nix::sys::signal::Signal::SIGUSR2).unwrap();
let both_reopened = async {
while runner.reopens(0) == 0 || runner.reopens(1) == 0 {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(10), both_reopened)
.await
.expect("SIGUSR2 must reopen every sheep's log files");
ctx.shutdown();
tokio::time::timeout(Duration::from_secs(10), run)
.await
.unwrap()
.unwrap()
.unwrap();
}
}