use core::time::Duration;
use std::collections::HashMap;
use std::fs;
use std::io;
#[cfg(unix)]
use std::os::fd::{AsRawFd, OwnedFd, RawFd};
#[cfg(unix)]
use std::os::unix::process::ExitStatusExt as _;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
#[cfg(unix)]
use command_fds::{CommandFdExt, FdMapping};
#[cfg(unix)]
use nix::sys::signal::{self, Signal};
#[cfg(unix)]
use nix::unistd::Pid;
use shep_core::signals::OperatorSignal;
use tokio::io::{
AsyncBufReadExt as _, AsyncRead, AsyncWrite, AsyncWriteExt as _, BufReader, BufWriter, Lines,
};
#[cfg(unix)]
use tokio::net::UnixStream;
use tokio::process::{Child, Command};
use tokio::sync::mpsc;
use tokio::time::{Instant, sleep_until, timeout};
#[cfg(unix)]
use crate::boot::DIR_MODE;
use crate::channel::{CHANNEL_VERSION, ChildMessage, ShepherdMessage};
#[cfg(unix)]
use crate::runner::{AdoptSpec, AdoptedReaper};
use crate::runner::{
ExitOutcome, FlushError, LogCtl, LogLine, Preflight, ProcIo, ProcessRunner, ReopenError,
RunnerError, RunningProcess, SpawnSpec, StdinWrite, StopSignal, check_log_ancestry,
open_log_path,
};
const CHANNEL_CAPACITY: usize = 32;
const LOG_BUFFER: usize = 8 * 1024;
const IDLE_FLUSH: Duration = Duration::from_millis(50);
use shep_core::logstamp::stamp_into;
#[cfg(unix)]
const READ_BUFFER: usize = 8 * 1024;
const FINAL_DRAIN: Duration = Duration::from_millis(100);
#[derive(Debug, Default)]
pub struct TokioRunner;
#[cfg(windows)]
const KILL_TREE_EXIT_CODE: u32 = 137;
#[cfg(windows)]
static NEXT_CHANNEL_ID: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
impl TokioRunner {
#[must_use]
pub fn new() -> Self {
Self
}
}
#[derive(Debug)]
pub struct TokioProc {
pid: u32,
proc: Supervised,
#[cfg(windows)]
job: crate::sys_windows::Job,
}
#[derive(Debug)]
enum Supervised {
Spawned(Child),
#[cfg(unix)]
Adopted(Arc<AdoptedReaper>),
}
impl RunningProcess for TokioProc {
fn pid(&self) -> u32 {
self.pid
}
async fn wait(&mut self) -> ExitOutcome {
let child = match &mut self.proc {
Supervised::Spawned(child) => child,
#[cfg(unix)]
Supervised::Adopted(reaper) => {
return match reaper.wait(self.pid).await {
Ok(outcome) => outcome,
Err(error) => {
tracing::error!(pid = self.pid, %error, "adopted process wait failed");
ExitOutcome {
code: None,
signal: None,
}
}
};
}
};
match child.wait().await {
Ok(status) => ExitOutcome {
code: status.code(),
#[cfg(windows)]
signal: None,
#[cfg(unix)]
signal: status.signal(),
},
Err(error) => {
tracing::error!(pid = self.pid, %error, "process wait() failed");
ExitOutcome {
code: None,
signal: None,
}
}
}
}
#[cfg(unix)]
fn signal(&mut self, sig: StopSignal) -> Result<(), RunnerError> {
signal_group(self.pid, to_nix_signal(sig))
}
#[cfg(windows)]
fn signal(&mut self, sig: StopSignal) -> Result<(), RunnerError> {
Err(RunnerError::SignalFailed(format!(
"Windows cannot deliver {sig:?} to another process; \
an app that needs a graceful stop must opt into the shepherd \
channel (shutdown_with_message), otherwise the stop escalates \
to a forced termination after kill_timeout"
)))
}
#[cfg(unix)]
fn kill_tree(&mut self) -> Result<(), RunnerError> {
signal_group(self.pid, Signal::SIGKILL)
}
#[cfg(windows)]
fn kill_tree(&mut self) -> Result<(), RunnerError> {
self.job
.terminate(KILL_TREE_EXIT_CODE)
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
#[cfg(windows)]
fn signal_process(&mut self, sig: OperatorSignal) -> Result<(), RunnerError> {
let Supervised::Spawned(child) = &mut self.proc;
match sig {
OperatorSignal::Kill => child
.start_kill()
.map_err(|error| RunnerError::SignalFailed(error.to_string())),
other => Err(RunnerError::SignalFailed(format!(
"Windows has no way to deliver {other:?} to another process; \
only SIGKILL is available here"
))),
}
}
#[cfg(unix)]
fn signal_process(&mut self, sig: OperatorSignal) -> Result<(), RunnerError> {
let pid = i32::try_from(self.pid)
.ok()
.filter(|pid| *pid > 0)
.ok_or_else(|| {
RunnerError::SignalFailed(format!(
"pid {} is not a signallable process id",
self.pid
))
})?;
signal::kill(Pid::from_raw(pid), to_nix_operator_signal(sig))
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
}
#[cfg(unix)]
pub(crate) fn signal_group(pid: u32, sig: Signal) -> Result<(), RunnerError> {
let pid = i32::try_from(pid)
.ok()
.filter(|pid| *pid > 0)
.ok_or_else(|| {
RunnerError::SignalFailed(format!("pid {pid} is not a signallable process id"))
})?;
signal::kill(Pid::from_raw(-pid), sig)
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
#[cfg(unix)]
fn to_nix_signal(sig: StopSignal) -> Signal {
match sig {
StopSignal::Term => Signal::SIGTERM,
StopSignal::Int => Signal::SIGINT,
StopSignal::Quit => Signal::SIGQUIT,
StopSignal::Usr2 => Signal::SIGUSR2,
StopSignal::Kill => Signal::SIGKILL,
}
}
#[cfg(unix)]
fn to_nix_operator_signal(sig: OperatorSignal) -> Signal {
match sig {
OperatorSignal::Hup => Signal::SIGHUP,
OperatorSignal::Int => Signal::SIGINT,
OperatorSignal::Quit => Signal::SIGQUIT,
OperatorSignal::Term => Signal::SIGTERM,
OperatorSignal::Usr1 => Signal::SIGUSR1,
OperatorSignal::Usr2 => Signal::SIGUSR2,
OperatorSignal::Winch => Signal::SIGWINCH,
OperatorSignal::Cont => Signal::SIGCONT,
OperatorSignal::Kill => Signal::SIGKILL,
}
}
fn what_exec_will_find(spec: &SpawnSpec) -> Preflight {
if spec.program.is_empty() {
return Preflight::Unknown;
}
let program = Path::new(&spec.program);
if spec.program.contains('/') || spec.program.contains(std::path::MAIN_SEPARATOR) {
let full = if program.is_absolute() {
program.to_path_buf()
} else {
match &spec.cwd {
Some(cwd) => cwd.join(program),
None => return Preflight::Unknown,
}
};
if !definitely_absent(&full) {
return Preflight::Unknown;
}
return Preflight::Impossible(format!("no such file: {}", full.display()));
}
let Some(path) = spec.env.get("PATH").filter(|value| !value.is_empty()) else {
return Preflight::Unknown;
};
for dir in std::env::split_paths(path).filter(|dir| !dir.as_os_str().is_empty()) {
match fs::metadata(dir.join(program)) {
Ok(_) => return Preflight::Unknown,
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(_) => return Preflight::Unknown,
}
}
Preflight::Doubtful(format!(
"`{}` is not on the shepherd's PATH ({})",
spec.program,
summarise_path(path)
))
}
const PATH_ENTRIES_IN_MESSAGE: usize = 4;
const PATH_LIST_SEPARATOR: char = if cfg!(windows) { ';' } else { ':' };
fn summarise_path(path: &str) -> String {
let entries: Vec<&str> = path
.split(PATH_LIST_SEPARATOR)
.filter(|dir| !dir.is_empty())
.collect();
if entries.len() <= PATH_ENTRIES_IN_MESSAGE {
return path.to_string();
}
format!(
"{} and {} more entries",
entries[..PATH_ENTRIES_IN_MESSAGE].join(&PATH_LIST_SEPARATOR.to_string()),
entries.len() - PATH_ENTRIES_IN_MESSAGE,
)
}
fn definitely_absent(path: &Path) -> bool {
matches!(fs::metadata(path), Err(err) if err.kind() == io::ErrorKind::NotFound)
}
impl ProcessRunner for TokioRunner {
type Proc = TokioProc;
fn preflight(&self, spec: &SpawnSpec) -> Preflight {
what_exec_will_find(spec)
}
#[cfg(unix)]
fn adopt(&self, spec: AdoptSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
let AdoptSpec {
pid,
out_file,
err_file,
out_pipe,
err_pipe,
out_log,
err_log,
stdin_pipe,
channel,
reaper,
} = spec;
let (logs_tx, logs_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (log_ctl_tx, log_ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
let pipes = PipeFds {
out: out_pipe.as_ref().map(AsRawFd::as_raw_fd),
err: err_pipe.as_ref().map(AsRawFd::as_raw_fd),
stdin: stdin_pipe.as_ref().map(AsRawFd::as_raw_fd),
channel: channel.as_ref().map(AsRawFd::as_raw_fd),
};
spawn_log_pump(
out_pipe,
err_pipe,
carried_sink(out_file, out_log),
carried_sink(err_file, err_log),
logs_tx,
log_ctl_rx,
pipes,
);
let (from_child_tx, from_child) = mpsc::channel(CHANNEL_CAPACITY);
let (to_child, to_child_rx) = mpsc::channel(CHANNEL_CAPACITY);
if let Some(channel) = channel {
spawn_channel_pumps(channel, from_child_tx, to_child_rx);
} else {
drop(from_child_tx);
drop(to_child_rx);
}
let (to_stdin, to_stdin_rx) = mpsc::channel(CHANNEL_CAPACITY);
if let Some(stdin_pipe) = stdin_pipe {
spawn_stdin_pump(Some(stdin_pipe), to_stdin_rx);
} else {
drop(to_stdin_rx);
}
Ok((
TokioProc {
pid,
proc: Supervised::Adopted(reaper),
},
ProcIo {
logs: logs_rx,
from_child,
to_child,
log_ctl: log_ctl_tx,
to_stdin,
},
))
}
fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
let mut command = Command::new(&spec.program);
command.args(&spec.args);
if let Some(cwd) = &spec.cwd {
command.current_dir(cwd);
}
command.env_clear();
command.envs(&spec.env);
command.stdin(if spec.stdin {
Stdio::piped()
} else {
Stdio::null()
});
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
#[cfg(unix)]
command.process_group(0);
#[cfg(windows)]
{
const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
command.creation_flags(CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW);
}
#[cfg(unix)]
if let Some(creds) = spec.credentials {
if let Some(gid) = creds.gid {
command.gid(gid);
}
command.uid(creds.uid);
}
#[cfg(windows)]
debug_assert!(
spec.credentials.is_none(),
"privilege::resolve must refuse user/group on Windows before a spawn is reached"
);
let (from_child_tx, from_child_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (to_child_tx, to_child_rx) = mpsc::channel(CHANNEL_CAPACITY);
#[cfg(windows)]
if spec.channel {
use shep_core::transport;
let mut nonce = [0_u8; 16];
getrandom::fill(&mut nonce).map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel pipe name: {error}"))
})?;
let pipe = std::path::PathBuf::from(format!(
r"\\.\pipe\shep-channel-{}-{}-{:032x}",
std::process::id(),
NEXT_CHANNEL_ID.fetch_add(1, core::sync::atomic::Ordering::Relaxed),
u128::from_ne_bytes(nonce)
));
let mut listener = transport::Listener::bind(&pipe).map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel pipe: {error}"))
})?;
command.env("SHEP_CHANNEL_PIPE", &pipe);
command.env("SHEP_CHANNEL_VERSION", CHANNEL_VERSION);
let from_child_tx = from_child_tx.clone();
tokio::spawn(async move {
let watcher = from_child_tx.clone();
tokio::select! {
accepted = listener.accept() => match accepted {
Ok(daemon_end) => {
spawn_channel_pumps(daemon_end, from_child_tx, to_child_rx);
}
Err(error) => {
tracing::warn!(%error, "shepherd channel accept failed");
}
},
() = watcher.closed() => {}
}
});
} else {
drop(to_child_rx);
}
#[cfg(unix)]
let mut channel_fd: Option<RawFd> = None;
#[cfg(unix)]
if spec.channel {
command.env("SHEP_CHANNEL_FD", "3");
command.env("SHEP_CHANNEL_VERSION", CHANNEL_VERSION);
let (daemon_end, child_end) = UnixStream::pair().map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel socketpair: {error}"))
})?;
let std_child_end = child_end.into_std().map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel into_std: {error}"))
})?;
std_child_end.set_nonblocking(false).map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel set_nonblocking: {error}"))
})?;
let child_fd = OwnedFd::from(std_child_end);
command
.as_std_mut()
.fd_mappings(vec![FdMapping {
parent_fd: child_fd,
child_fd: 3,
}])
.map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel fd mapping: {error}"))
})?;
channel_fd = Some(daemon_end.as_raw_fd());
spawn_channel_pumps(daemon_end, from_child_tx, to_child_rx);
} else {
drop(from_child_tx);
drop(to_child_rx);
}
let mut child = command
.spawn()
.map_err(|error| RunnerError::SpawnFailed(error.to_string()))?;
drop(command);
let pid = child.id().ok_or_else(|| {
RunnerError::SpawnFailed("child exited before its pid could be read".to_string())
})?;
#[cfg(windows)]
let job = {
let job = crate::sys_windows::Job::create().map_err(|error| {
RunnerError::SpawnFailed(format!("job object for {}: {error}", spec.name))
})?;
let handle = child.raw_handle().ok_or_else(|| {
RunnerError::SpawnFailed("child exited before it could be contained".to_string())
})?;
if let Err(error) = job.assign(handle) {
let _ = child.start_kill();
return Err(RunnerError::SpawnFailed(format!(
"could not contain {} in a job object: {error}",
spec.name
)));
}
job
};
let (logs_tx, logs_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (log_ctl_tx, log_ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
#[cfg(unix)]
let pipes = PipeFds {
out: child.stdout.as_ref().map(AsRawFd::as_raw_fd),
err: child.stderr.as_ref().map(AsRawFd::as_raw_fd),
stdin: child.stdin.as_ref().map(AsRawFd::as_raw_fd),
channel: channel_fd,
};
#[cfg(not(unix))]
let pipes = PipeFds;
spawn_log_pump(
child.stdout.take(),
child.stderr.take(),
LogSink::Path(spec.out_file.clone()),
LogSink::Path(spec.err_file.clone()),
logs_tx,
log_ctl_rx,
pipes,
);
let (to_stdin_tx, to_stdin_rx) = mpsc::channel(CHANNEL_CAPACITY);
if spec.stdin {
spawn_stdin_pump(child.stdin.take(), to_stdin_rx);
} else {
drop(to_stdin_rx);
}
let io = ProcIo {
logs: logs_rx,
from_child: from_child_rx,
to_child: to_child_tx,
log_ctl: log_ctl_tx,
to_stdin: to_stdin_tx,
};
Ok((
TokioProc {
pid,
proc: Supervised::Spawned(child),
#[cfg(windows)]
job,
},
io,
))
}
}
#[cfg(unix)]
fn carried_sink(path: PathBuf, handle: Option<tokio::fs::File>) -> LogSink {
match handle {
Some(file) => LogSink::Carried(path, file),
None => LogSink::Path(path),
}
}
#[derive(Debug)]
struct LogFile<W = tokio::fs::File> {
path: PathBuf,
handle: Option<BufWriter<W>>,
record: Arc<tokio::sync::Mutex<()>>,
buffered_since: Option<Instant>,
stamp: String,
}
enum LogSink {
Path(PathBuf),
#[cfg(unix)]
Carried(PathBuf, tokio::fs::File),
}
impl LogFile<tokio::fs::File> {
async fn from_sink(sink: LogSink) -> Self {
match sink {
LogSink::Path(path) => Self::open(path).await,
#[cfg(unix)]
LogSink::Carried(path, file) => Self::from_file(path, file),
}
}
#[cfg(unix)]
fn from_file(path: PathBuf, file: tokio::fs::File) -> Self {
Self {
record: record_lock(&path),
path,
handle: Some(BufWriter::with_capacity(LOG_BUFFER, file)),
buffered_since: None,
stamp: String::new(),
}
}
async fn open(path: PathBuf) -> Self {
let handle = open_append(&path)
.await
.ok()
.map(|file| BufWriter::with_capacity(LOG_BUFFER, file));
Self {
record: record_lock(&path),
path,
handle,
buffered_since: None,
stamp: String::new(),
}
}
async fn reopen(&mut self) -> Result<(), ReopenError> {
self.buffered_since = None;
if let Some(handle) = self.handle.as_mut() {
let _record = self.record.lock().await;
if let Err(error) = handle.flush().await {
tracing::error!(path = ?self.path, %error, "log file flush failed");
}
}
drop(self.handle.take());
match open_append(&self.path).await {
Ok(handle) => {
self.handle = Some(BufWriter::with_capacity(LOG_BUFFER, handle));
Ok(())
}
Err(error) => Err(ReopenError {
message: format!("{}: {error}", self.path.display()),
}),
}
}
#[cfg(unix)]
fn raw_fd(&self) -> Option<RawFd> {
self.handle
.as_ref()
.map(|handle| handle.get_ref().as_raw_fd())
}
}
pub(crate) fn record_lock(path: &Path) -> Arc<tokio::sync::Mutex<()>> {
static LOCKS: std::sync::OnceLock<
std::sync::Mutex<HashMap<PathBuf, Arc<tokio::sync::Mutex<()>>>>,
> = std::sync::OnceLock::new();
let mut locks = LOCKS
.get_or_init(std::sync::Mutex::default)
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Arc::clone(locks.entry(path.to_path_buf()).or_default())
}
impl<W: AsyncWrite + Unpin> LogFile<W> {
async fn append(&mut self, line: &str) {
let Some(handle) = self.handle.as_mut() else {
return;
};
self.stamp.clear();
stamp_into(&mut self.stamp);
self.stamp.push_str(line);
self.stamp.push('\n');
let written = {
let _record = self.record.lock().await;
handle.write_all(self.stamp.as_bytes()).await
};
self.buffered_since.get_or_insert_with(Instant::now);
if let Err(error) = written {
tracing::error!(path = ?self.path, %error, "log file append failed");
}
}
async fn flush(&mut self) -> Result<(), FlushError> {
self.buffered_since = None;
let Some(handle) = self.handle.as_mut() else {
return Ok(());
};
let _record = self.record.lock().await;
handle.flush().await.map_err(|error| FlushError {
message: format!("{}: {error}", self.path.display()),
})
}
}
#[cfg(unix)]
#[derive(Debug, Clone, Copy, Default)]
struct PipeFds {
out: Option<RawFd>,
err: Option<RawFd>,
stdin: Option<RawFd>,
channel: Option<RawFd>,
}
#[cfg(not(unix))]
#[derive(Debug, Clone, Copy, Default)]
struct PipeFds;
#[derive(Debug)]
struct LogFiles {
out: LogFile,
err: LogFile,
pipes: PipeFds,
#[cfg(unix)]
parked: bool,
}
impl LogFiles {
fn stream(&mut self, err: bool) -> &mut LogFile {
if err { &mut self.err } else { &mut self.out }
}
fn reading(&self) -> bool {
#[cfg(unix)]
{
!self.parked
}
#[cfg(not(unix))]
{
true
}
}
fn flush_deadline(&self) -> Option<Instant> {
let oldest = match (self.out.buffered_since, self.err.buffered_since) {
(Some(out), Some(err)) => out.min(err),
(Some(only), None) | (None, Some(only)) => only,
(None, None) => return None,
};
Some(oldest + IDLE_FLUSH)
}
async fn flush_idle(&mut self) {
for file in [&mut self.out, &mut self.err] {
if let Err(error) = file.flush().await {
tracing::error!(%error, "log file idle flush failed");
}
}
}
#[cfg_attr(
not(unix),
allow(
unused_variables,
reason = "`streams` is read only by `ReportFds`, and that variant is unix-only"
)
)]
async fn serve<O, E>(&mut self, ctl: LogCtl, streams: &mut Streams<O, E>)
where
O: AsyncRead + Unpin,
E: AsyncRead + Unpin,
{
match ctl {
LogCtl::Reopen { done } => {
let mut failures = Vec::new();
if let Err(error) = self.out.reopen().await {
failures.push(error.message);
}
if let Err(error) = self.err.reopen().await {
failures.push(error.message);
}
let result = if failures.is_empty() {
Ok(())
} else {
Err(ReopenError {
message: failures.join(", "),
})
};
let _ = done.send(result);
}
LogCtl::Flush { done } => {
let mut failures = Vec::new();
if let Err(error) = self.out.flush().await {
failures.push(error.message);
}
if let Err(error) = self.err.flush().await {
failures.push(error.message);
}
let result = if failures.is_empty() {
Ok(())
} else {
Err(FlushError {
message: failures.join(", "),
})
};
let _ = done.send(result);
}
#[cfg(unix)]
LogCtl::ReportFds { done } => {
self.parked = true;
drain_ready(&mut streams.out, &mut self.out).await;
drain_ready(&mut streams.err, &mut self.err).await;
for file in [&mut self.out, &mut self.err] {
if let Err(error) = file.flush().await {
tracing::error!(%error, "log flush before a handover report failed");
}
}
let _ = done.send(crate::handover::CarriedFds {
out_pipe: self.pipes.out,
err_pipe: self.pipes.err,
out_log: self.out.raw_fd(),
err_log: self.err.raw_fd(),
stdin: self.pipes.stdin,
channel: self.pipes.channel,
});
}
#[cfg(unix)]
LogCtl::Resume => self.parked = false,
}
}
}
enum AfterLine {
KeepReading,
StreamEnded,
LogsClosed,
}
async fn deliver_line<O, E>(
result: io::Result<Option<String>>,
err: bool,
files: &mut LogFiles,
logs_tx: &mpsc::Sender<LogLine>,
ctl_rx: &mut mpsc::Receiver<LogCtl>,
streams: &mut Streams<O, E>,
) -> AfterLine
where
O: AsyncRead + Unpin,
E: AsyncRead + Unpin,
{
match result {
Ok(Some(line)) => {
files.stream(err).append(&line).await;
let Some(slot) = reserve_slot(logs_tx, files, ctl_rx, streams).await else {
return AfterLine::LogsClosed;
};
slot.send(LogLine { err, line });
AfterLine::KeepReading
}
Ok(None) => AfterLine::StreamEnded, Err(error) => {
tracing::error!(path = ?files.stream(err).path, %error, "log stream read failed");
AfterLine::StreamEnded
}
}
}
async fn reserve_slot<'tx, O, E>(
logs_tx: &'tx mpsc::Sender<LogLine>,
files: &mut LogFiles,
ctl_rx: &mut mpsc::Receiver<LogCtl>,
streams: &mut Streams<O, E>,
) -> Option<mpsc::Permit<'tx, LogLine>>
where
O: AsyncRead + Unpin,
E: AsyncRead + Unpin,
{
loop {
let flush_at = files.flush_deadline();
tokio::select! {
slot = logs_tx.reserve() => return slot.ok(),
ctl = ctl_rx.recv() => match ctl {
Some(ctl) => files.serve(ctl, streams).await,
None => return logs_tx.reserve().await.ok(),
},
() = sleep_until(flush_at.unwrap_or_else(Instant::now)), if flush_at.is_some() => {
files.flush_idle().await;
}
}
}
}
async fn next_line<R>(lines: &mut Option<Lines<BufReader<R>>>) -> io::Result<Option<String>>
where
R: AsyncRead + Unpin,
{
match lines {
Some(lines) => lines.next_line().await,
None => core::future::pending().await,
}
}
#[cfg(unix)]
fn with_read_buffer<R: AsyncRead>(reader: R) -> BufReader<R> {
BufReader::with_capacity(READ_BUFFER, reader)
}
#[cfg(not(unix))]
fn with_read_buffer<R: AsyncRead>(reader: R) -> BufReader<R> {
BufReader::new(reader)
}
struct Streams<O, E> {
out: Option<Lines<BufReader<O>>>,
err: Option<Lines<BufReader<E>>>,
}
#[cfg(unix)]
async fn drain_ready<R>(lines: &mut Option<Lines<BufReader<R>>>, file: &mut LogFile)
where
R: AsyncRead + Unpin,
{
let Some(reader) = lines.as_mut() else {
return;
};
while reader.get_ref().buffer().contains(&b'\n') {
let Ok(Some(line)) = reader.next_line().await else {
return;
};
file.append(&line).await;
}
}
async fn final_drain<O, E>(files: &mut LogFiles, streams: &mut Streams<O, E>)
where
O: AsyncRead + Unpin,
E: AsyncRead + Unpin,
{
let drained = timeout(FINAL_DRAIN, async {
while streams.out.is_some() || streams.err.is_some() {
tokio::select! {
result = next_line(&mut streams.out) => {
match result {
Ok(Some(line)) => files.stream(false).append(&line).await,
Ok(None) | Err(_) => streams.out = None,
}
}
result = next_line(&mut streams.err) => {
match result {
Ok(Some(line)) => files.stream(true).append(&line).await,
Ok(None) | Err(_) => streams.err = None,
}
}
}
}
})
.await;
if drained.is_err() {
tracing::debug!("a pump left the pipe still open at its sheep task's exit");
}
}
fn spawn_log_pump<O, E>(
stdout: Option<O>,
stderr: Option<E>,
out_sink: LogSink,
err_sink: LogSink,
logs_tx: mpsc::Sender<LogLine>,
mut ctl_rx: mpsc::Receiver<LogCtl>,
pipes: PipeFds,
) where
O: AsyncRead + Unpin + Send + 'static,
E: AsyncRead + Unpin + Send + 'static,
{
tokio::spawn(async move {
let mut files = LogFiles {
out: LogFile::from_sink(out_sink).await,
err: LogFile::from_sink(err_sink).await,
pipes,
#[cfg(unix)]
parked: false,
};
let mut streams = Streams {
out: stdout.map(|reader| with_read_buffer(reader).lines()),
err: stderr.map(|reader| with_read_buffer(reader).lines()),
};
while streams.out.is_some() || streams.err.is_some() {
let flush_at = files.flush_deadline();
tokio::select! {
result = next_line(&mut streams.out), if files.reading() => {
let after =
deliver_line(result, false, &mut files, &logs_tx, &mut ctl_rx, &mut streams)
.await;
match after {
AfterLine::KeepReading => {}
AfterLine::StreamEnded => {
streams.out = None;
#[cfg(unix)]
{
files.pipes.out = None;
}
}
AfterLine::LogsClosed => break,
}
}
result = next_line(&mut streams.err), if files.reading() => {
let after =
deliver_line(result, true, &mut files, &logs_tx, &mut ctl_rx, &mut streams)
.await;
match after {
AfterLine::KeepReading => {}
AfterLine::StreamEnded => {
streams.err = None;
#[cfg(unix)]
{
files.pipes.err = None;
}
}
AfterLine::LogsClosed => break,
}
}
ctl = ctl_rx.recv() => {
match ctl {
Some(ctl) => files.serve(ctl, &mut streams).await,
None => break, }
}
() = logs_tx.closed() => break,
() = sleep_until(flush_at.unwrap_or_else(Instant::now)), if flush_at.is_some() => {
files.flush_idle().await;
}
}
}
if files.reading() {
final_drain(&mut files, &mut streams).await;
}
files.flush_idle().await;
});
}
pub(crate) async fn open_append(path: &Path) -> io::Result<tokio::fs::File> {
check_log_ancestry(path)
.inspect_err(|error| tracing::error!(?path, %error, "log ancestry check failed"))?;
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
let mut builder = tokio::fs::DirBuilder::new();
builder.recursive(true);
#[cfg(unix)]
builder.mode(DIR_MODE);
if let Err(error) = builder.create(parent).await {
tracing::error!(?path, %error, "log directory create failed");
return Err(error);
}
}
let mut options = tokio::fs::OpenOptions::new();
options.create(true).append(true);
open_log_path(&mut options, path)
.await
.inspect_err(|error| tracing::error!(?path, %error, "log file open failed"))
}
fn spawn_stdin_pump<W>(stdin: Option<W>, mut rx: mpsc::Receiver<StdinWrite>)
where
W: AsyncWrite + Unpin + Send + 'static,
{
tokio::spawn(async move {
let Some(mut stdin) = stdin else {
while let Some(StdinWrite { done, .. }) = rx.recv().await {
let _ = done.send(Err(RunnerError::WriteFailed(
"this child has no stdin pipe".to_string(),
)));
}
return;
};
while let Some(StdinWrite { line, done }) = rx.recv().await {
if done.is_closed() {
continue;
}
let mut bytes = line.into_bytes();
bytes.push(b'\n');
let result = match stdin.write_all(&bytes).await {
Ok(()) => stdin.flush().await,
Err(error) => Err(error),
};
let _ = done.send(result.map_err(|error| RunnerError::WriteFailed(error.to_string())));
}
});
}
fn spawn_channel_pumps<S>(
daemon_end: S,
from_child_tx: mpsc::Sender<ChildMessage>,
mut to_child_rx: mpsc::Receiver<ShepherdMessage>,
) where
S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
{
let (read_half, mut write_half) = tokio::io::split(daemon_end);
tokio::spawn(async move {
let mut lines = BufReader::new(read_half).lines();
loop {
match lines.next_line().await {
Ok(Some(line)) => match serde_json::from_str::<ChildMessage>(&line) {
Ok(msg) => {
if from_child_tx.send(msg).await.is_err() {
break; }
}
Err(error) => {
tracing::warn!(%line, %error, "malformed shepherd-channel frame");
}
},
Ok(None) => break, Err(error) => {
tracing::error!(%error, "shepherd-channel read failed");
break;
}
}
}
});
tokio::spawn(async move {
while let Some(msg) = to_child_rx.recv().await {
let mut line = match serde_json::to_string(&msg) {
Ok(json) => json,
Err(error) => {
tracing::error!(%error, "shepherd message encode failed");
continue;
}
};
line.push('\n');
if write_half.write_all(line.as_bytes()).await.is_err() {
break; }
}
});
}
#[cfg(test)]
mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn narration_cannot_tear_a_line_the_pump_is_writing() {
let dir = tempfile::tempdir().expect("a temp dir");
let path = dir.path().join("torn.log");
let long = "x".repeat(LOG_BUFFER + 512);
let pump = {
let path = path.clone();
let long = long.clone();
tokio::spawn(async move {
let mut log = LogFile::open(path).await;
for i in 0..1200 {
log.append(&format!("{i} {long}")).await;
if i % 4 == 0 {
tokio::task::yield_now().await;
}
}
log.flush().await.expect("the pump's own flush");
})
};
let narrator = {
let path = path.clone();
tokio::spawn(async move {
for i in 0..600 {
let mut written = String::new();
stamp_into(&mut written);
written.push_str(&format!("[shep] narration {i}\n"));
let file = open_append(&path).await.expect("the narration handle");
let _record = record_lock(&path).lock_owned().await;
let mut file = file;
use tokio::io::AsyncWriteExt as _;
file.write_all(written.as_bytes()).await.expect("write");
file.flush().await.expect("flush");
}
})
};
pump.await.expect("the pump task");
narrator.await.expect("the narration task");
let text = std::fs::read_to_string(&path).expect("the log is readable");
let mut lines = 0_usize;
for line in text.lines() {
lines += 1;
let once = shep_core::logstamp::strip(line);
assert_ne!(
once, line,
"a line reached the file with no stamp: {line:.80}"
);
assert_eq!(
shep_core::logstamp::strip(once),
once,
"a line carries two stamps, so a record was torn: {line:.80}"
);
}
assert_eq!(lines, 1800, "every record reached the file exactly once");
}
use core::pin::Pin;
use core::sync::atomic::{AtomicUsize, Ordering};
use core::task::{Context, Poll};
use std::collections::BTreeMap;
#[cfg(unix)]
use std::collections::BTreeSet;
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt as _;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::DuplexStream;
use tokio::sync::oneshot;
use tokio::time::timeout;
use super::*;
#[cfg(unix)]
use crate::handover::CarriedFds;
const PUMP_DEADLINE: Duration = Duration::from_secs(5);
const STREAM_BUFFER: usize = 4096;
struct PumpHarness<W = DuplexStream> {
dir: tempfile::TempDir,
out_path: PathBuf,
err_path: PathBuf,
out_writer: W,
err_writer: W,
logs: mpsc::Receiver<LogLine>,
ctl: mpsc::Sender<LogCtl>,
pipes: PipeFds,
}
impl PumpHarness {
fn start() -> Self {
let dir = tempfile::tempdir().unwrap();
let out_path = dir.path().join("out.log");
let err_path = dir.path().join("err.log");
let (out_writer, out_reader) = tokio::io::duplex(STREAM_BUFFER);
let (err_writer, err_reader) = tokio::io::duplex(STREAM_BUFFER);
let (logs_tx, logs) = mpsc::channel(CHANNEL_CAPACITY);
let (ctl, ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
spawn_log_pump(
Some(out_reader),
Some(err_reader),
LogSink::Path(out_path.clone()),
LogSink::Path(err_path.clone()),
logs_tx,
ctl_rx,
PipeFds::default(),
);
Self {
dir,
out_path,
err_path,
out_writer,
err_writer,
logs,
ctl,
pipes: PipeFds::default(),
}
}
}
#[cfg(unix)]
impl PumpHarness<tokio::net::unix::pipe::Sender> {
fn start_over_pipes() -> Self {
let dir = tempfile::tempdir().unwrap();
let out_path = dir.path().join("out.log");
let err_path = dir.path().join("err.log");
let (out_writer, out_reader) = tokio::net::unix::pipe::pipe().unwrap();
let (err_writer, err_reader) = tokio::net::unix::pipe::pipe().unwrap();
let pipes = PipeFds {
out: Some(out_reader.as_raw_fd()),
err: Some(err_reader.as_raw_fd()),
stdin: None,
channel: None,
};
let (logs_tx, logs) = mpsc::channel(CHANNEL_CAPACITY);
let (ctl, ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
spawn_log_pump(
Some(out_reader),
Some(err_reader),
LogSink::Path(out_path.clone()),
LogSink::Path(err_path.clone()),
logs_tx,
ctl_rx,
pipes,
);
Self {
dir,
out_path,
err_path,
out_writer,
err_writer,
logs,
ctl,
pipes,
}
}
}
impl<W: AsyncWrite + Unpin> PumpHarness<W> {
#[cfg(unix)]
async fn report_fds(&self) -> CarriedFds {
let (done, ack) = oneshot::channel();
self.ctl
.send(LogCtl::ReportFds { done })
.await
.expect("the pump must still be reading its control channel");
timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement")
}
#[cfg(unix)]
async fn resume(&self) {
self.ctl
.send(LogCtl::Resume)
.await
.expect("a parked pump must still be reading its control channel");
}
async fn feed(&mut self, err: bool, line: &str) {
let writer = if err {
&mut self.err_writer
} else {
&mut self.out_writer
};
writer
.write_all(format!("{line}\n").as_bytes())
.await
.unwrap();
let observed = timeout(PUMP_DEADLINE, self.logs.recv())
.await
.expect("the pump must forward a line it has read")
.expect("the pump must not end while its streams are open");
assert_eq!(
observed,
LogLine {
err,
line: line.to_string()
}
);
}
async fn reopen(&self) {
let outcome = self.reopen_for_answer().await;
assert_eq!(outcome, Ok(()), "this reopen must have worked");
}
async fn reopen_for_answer(&self) -> Result<(), ReopenError> {
let (done, ack) = oneshot::channel();
self.ctl
.send(LogCtl::Reopen { done })
.await
.expect("the pump must still be reading its control channel");
timeout(PUMP_DEADLINE, ack)
.await
.expect("a reopen must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement")
}
async fn flush(&self) {
let (done, ack) = oneshot::channel();
self.ctl
.send(LogCtl::Flush { done })
.await
.expect("the pump must still be reading its control channel");
let outcome = timeout(PUMP_DEADLINE, ack)
.await
.expect("a flush must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
assert_eq!(outcome, Ok(()), "this flush must have worked");
}
}
use shep_core::logstamp::LOG_STAMP_BYTES;
fn log_text(path: &Path) -> String {
unstamped(&fs::read_to_string(path).unwrap())
}
fn unstamped(text: &str) -> String {
let mut out = String::new();
for line in text.lines() {
out.push_str(shep_core::logstamp::strip(line));
out.push('\n');
}
out
}
async fn assert_file_settles(path: &Path, expected: &str) {
let settled = timeout(PUMP_DEADLINE, async {
while unstamped(&fs::read_to_string(path).unwrap_or_default()) != expected {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await;
assert!(
settled.is_ok(),
"{}: expected {expected:?}, found {:?}",
path.display(),
fs::read_to_string(path)
);
}
#[cfg(unix)]
fn is_open(fd: RawFd) -> bool {
nix::fcntl::fcntl(fd, nix::fcntl::FcntlArg::F_GETFD).is_ok()
}
#[cfg(unix)]
#[tokio::test]
async fn a_pump_reports_the_descriptors_it_holds() {
let pump = PumpHarness::start_over_pipes();
let fds = pump.report_fds().await;
assert_eq!(fds.out_pipe, pump.pipes.out, "stdout's read end");
assert_eq!(fds.err_pipe, pump.pipes.err, "stderr's read end");
assert!(
fds.out_log.is_some() && fds.err_log.is_some(),
"a pump that opened both log files must name both handles: {fds:?}"
);
let named: Vec<RawFd> = fds.all().into_iter().flatten().collect();
assert_eq!(named.len(), 4, "a running sheep has all four: {fds:?}");
for fd in &named {
assert!(
is_open(*fd),
"the blob would name a closed descriptor: {fd}"
);
}
let distinct: BTreeSet<RawFd> = named.iter().copied().collect();
assert_eq!(distinct.len(), 4, "four descriptors, four numbers: {fds:?}");
}
#[cfg(unix)]
#[tokio::test]
async fn both_pipes_filled_to_capacity_drain_inside_the_budget() {
let pump = PumpHarness::start_over_pipes();
let line = format!("{}\n", "x".repeat(63));
let out_lines = fill_pipe(&pump.out_writer, &line).await;
let err_lines = fill_pipe(&pump.err_writer, &line).await;
assert!(
out_lines > 0 && err_lines > 0,
"neither pipe took a whole line: stdout={out_lines} stderr={err_lines}"
);
drop(pump.out_writer);
drop(pump.err_writer);
drop(pump.logs);
for (path, want, stream) in [
(&pump.out_path, out_lines, "stdout"),
(&pump.err_path, err_lines, "stderr"),
] {
let settled = timeout(PUMP_DEADLINE, async {
while fs::read_to_string(path).unwrap_or_default().lines().count() < want {
tokio::time::sleep(Duration::from_millis(2)).await;
}
})
.await;
assert!(
settled.is_ok(),
"a full {stream} pipe did not drain inside {FINAL_DRAIN:?}: {want} lines \
written, {} landed",
fs::read_to_string(path).unwrap_or_default().lines().count()
);
}
}
const RACE_ATTEMPTS: usize = 32;
#[cfg(unix)]
async fn fill_pipe(writer: &tokio::net::unix::pipe::Sender, line: &str) -> usize {
timeout(PUMP_DEADLINE, writer.writable())
.await
.expect("an empty pipe is writable")
.expect("the pipe is open");
let mut whole = 0usize;
loop {
match writer.try_write(line.as_bytes()) {
Ok(written) if written == line.len() => whole += 1,
Ok(_) => return whole,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => return whole,
Err(error) => {
panic!("the fill failed for a reason that is not a full pipe: {error}")
}
}
}
}
#[tokio::test]
async fn a_last_line_written_before_the_sheep_task_lets_go_reaches_the_file() {
for attempt in 0..RACE_ATTEMPTS {
let mut pump = PumpHarness::start();
pump.out_writer.write_all(b"last-words\n").await.unwrap();
drop(pump.out_writer);
drop(pump.err_writer);
drop(pump.logs);
let settled = timeout(PUMP_DEADLINE, async {
while !fs::read_to_string(&pump.out_path)
.unwrap_or_default()
.contains("last-words")
{
tokio::time::sleep(Duration::from_millis(2)).await;
}
})
.await;
assert!(
settled.is_ok(),
"attempt {attempt}: the pump dropped the line it had not read yet, \
leaving {:?}",
fs::read_to_string(&pump.out_path)
);
}
}
#[tokio::test]
async fn a_last_line_survives_the_control_channel_closing_with_the_logs() {
for attempt in 0..RACE_ATTEMPTS {
let mut pump = PumpHarness::start();
pump.out_writer.write_all(b"last-words\n").await.unwrap();
drop(pump.out_writer);
drop(pump.err_writer);
drop(pump.ctl);
drop(pump.logs);
let settled = timeout(PUMP_DEADLINE, async {
while !fs::read_to_string(&pump.out_path)
.unwrap_or_default()
.contains("last-words")
{
tokio::time::sleep(Duration::from_millis(2)).await;
}
})
.await;
assert!(
settled.is_ok(),
"attempt {attempt}: the pump took an exit that skips the drain, \
leaving {:?}",
fs::read_to_string(&pump.out_path)
);
}
}
#[cfg(unix)]
#[tokio::test]
async fn two_pumps_on_one_log_path_report_different_numbers() {
let dir = tempfile::tempdir().unwrap();
let out_path = dir.path().join("merged-out.log");
let err_path = dir.path().join("merged-err.log");
let mut writers = Vec::new();
let mut reports = Vec::new();
for _ in 0..2 {
let (out_writer, out_reader) = tokio::net::unix::pipe::pipe().unwrap();
let (err_writer, err_reader) = tokio::net::unix::pipe::pipe().unwrap();
let pipes = PipeFds {
out: Some(out_reader.as_raw_fd()),
err: Some(err_reader.as_raw_fd()),
stdin: None,
channel: None,
};
let (logs_tx, _logs) = mpsc::channel(CHANNEL_CAPACITY);
let (ctl, ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
spawn_log_pump(
Some(out_reader),
Some(err_reader),
LogSink::Path(out_path.clone()),
LogSink::Path(err_path.clone()),
logs_tx,
ctl_rx,
pipes,
);
let (done, ack) = oneshot::channel();
ctl.send(LogCtl::ReportFds { done }).await.unwrap();
let fds = timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
writers.push((out_writer, err_writer, ctl, _logs));
reports.push(fds);
}
let named: Vec<RawFd> = reports
.iter()
.flat_map(|fds| fds.all().into_iter().flatten())
.collect();
assert_eq!(
named.len(),
8,
"two running instances, four descriptors each: {reports:?}"
);
let distinct: BTreeSet<RawFd> = named.iter().copied().collect();
assert_eq!(
distinct.len(),
8,
"a repeat here refuses the whole handover, not just this app: {reports:?}"
);
assert_ne!(
reports[0].out_log, reports[1].out_log,
"two instances sharing one log file must still hold two handles"
);
assert_ne!(reports[0].err_log, reports[1].err_log);
drop(writers);
}
#[cfg(unix)]
#[tokio::test]
async fn a_report_after_a_stream_ends_names_no_descriptor_for_it() {
let mut pump = PumpHarness::start_over_pipes();
pump.feed(false, "before-the-eof").await;
drop(pump.out_writer);
let_the_pump_settle().await;
let (done, ack) = oneshot::channel();
pump.ctl
.send(LogCtl::ReportFds { done })
.await
.expect("a pump with one live stream still reads its control channel");
let fds = timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
assert_eq!(
fds.out_pipe, None,
"stdout is at EOF and its descriptor is gone: {fds:?}"
);
assert_eq!(
fds.err_pipe, pump.pipes.err,
"stderr is still live and keeps its own number: {fds:?}"
);
assert!(
fds.out_log.is_some() && fds.err_log.is_some(),
"both log handles are held whichever stream ended: {fds:?}"
);
}
#[cfg(unix)]
#[tokio::test]
async fn reporting_flushes_first() {
let mut pump = PumpHarness::start_over_pipes();
pump.feed(false, "before-the-blob").await;
pump.feed(true, "and-on-stderr").await;
let _ = pump.report_fds().await;
assert_eq!(log_text(&pump.out_path), "before-the-blob\n");
assert_eq!(log_text(&pump.err_path), "and-on-stderr\n");
}
#[cfg(unix)]
const STILL_WINDOWS: u32 = 6;
#[cfg(unix)]
const BURST: u32 = 60;
#[cfg(unix)]
async fn assert_file_holds_still(path: &Path, expected: &str) {
for window in 1..=STILL_WINDOWS {
tokio::time::sleep(IDLE_FLUSH).await;
assert_eq!(
log_text(path),
expected,
"{}: a parked pump wrote during flush window {window}",
path.display()
);
}
}
#[cfg(unix)]
async fn wait_for_a_full_logs_channel(logs: &mpsc::Receiver<LogLine>) {
let filled = timeout(PUMP_DEADLINE, async {
while logs.len() < CHANNEL_CAPACITY {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await;
assert!(
filled.is_ok(),
"the pump must fall behind a burst it cannot forward"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_pump_that_has_reported_stops_reading_until_it_is_resumed() {
let mut pump = PumpHarness::start();
pump.feed(false, "before-the-report").await;
let _ = pump.report_fds().await;
pump.out_writer
.write_all(b"after-1\nafter-2\n")
.await
.unwrap();
assert_file_holds_still(&pump.out_path, "before-the-report\n").await;
pump.resume().await;
assert_file_settles(&pump.out_path, "before-the-report\nafter-1\nafter-2\n").await;
}
#[cfg(unix)]
#[tokio::test]
async fn a_report_lands_what_the_reader_was_holding() {
let mut pump = PumpHarness::start();
let burst: String = (1..=BURST).map(|n| format!("{n}\n")).collect();
pump.out_writer.write_all(burst.as_bytes()).await.unwrap();
wait_for_a_full_logs_channel(&pump.logs).await;
let _ = pump.report_fds().await;
assert_eq!(
log_text(&pump.out_path),
burst,
"every line the reader held must be on disk once, in order, \
before the report is answered"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_report_leaves_in_the_pipe_everything_it_did_not_write() {
let dir = tempfile::tempdir().unwrap();
let out_path = dir.path().join("out.log");
let (reader, mut writer) = std::io::pipe().unwrap();
let inherited = reader.try_clone().unwrap();
let reader =
tokio::net::unix::pipe::Receiver::from_owned_fd(OwnedFd::from(reader)).unwrap();
let (logs_tx, logs) = mpsc::channel(CHANNEL_CAPACITY);
let (ctl, ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
let pipes = PipeFds {
out: Some(reader.as_raw_fd()),
err: None,
stdin: None,
channel: None,
};
spawn_log_pump(
Some(reader),
None::<DuplexStream>,
LogSink::Path(out_path.clone()),
LogSink::Path(dir.path().join("err.log")),
logs_tx,
ctl_rx,
pipes,
);
let lines = u32::try_from(SMALLEST_PIPE * 3 / 4 / RUN_LINE).unwrap();
let run: String = (1..=lines).map(|n| format!("{n:07}\n")).collect();
assert!(run.len() > READ_BUFFER, "the reader must fill");
std::io::Write::write_all(&mut writer, run.as_bytes()).unwrap();
wait_for_a_full_logs_channel(&logs).await;
let (done, ack) = oneshot::channel();
ctl.send(LogCtl::ReportFds { done }).await.unwrap();
timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
drop(writer);
let rest = read_to_eof(inherited).await;
let written = log_text(&out_path);
assert!(
run.starts_with(&written),
"the pump must write the run's own bytes, in order"
);
assert!(
run.ends_with(&rest),
"what is left in the pipe must be the run's own tail"
);
assert!(
written.len() + rest.len() >= run.len() - RUN_LINE,
"{} of {} bytes reached neither the file ({}) nor the pipe ({}): the report read \
them out of the pipe and then dropped them",
run.len() - written.len() - rest.len(),
run.len(),
written.len(),
rest.len()
);
}
#[cfg(unix)]
async fn read_to_eof(handle: std::io::PipeReader) -> String {
let mut out = Vec::new();
let mut buf = [0_u8; 4096];
let drained = timeout(PUMP_DEADLINE, async {
loop {
match std::io::Read::read(&mut &handle, &mut buf) {
Ok(0) => return,
Ok(n) => out.extend_from_slice(&buf[..n]),
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
tokio::time::sleep(Duration::from_millis(5)).await;
}
Err(error) => panic!("the successor's handle must be readable: {error}"),
}
}
})
.await;
assert!(
drained.is_ok(),
"the pipe must reach EOF once the sheep is gone"
);
String::from_utf8(out).expect("a pipe of ASCII lines")
}
#[cfg(unix)]
const SMALLEST_PIPE: usize = 16 * 1024;
#[cfg(unix)]
const RUN_LINE: usize = 8;
#[cfg(unix)]
#[tokio::test]
async fn a_report_drains_at_most_one_bufferful() {
let mut pump = PumpHarness::start_over_pipes();
let lines = u32::try_from(SMALLEST_PIPE * 3 / 4 / RUN_LINE).unwrap();
let run: String = (1..=lines).map(|n| format!("{n:07}\n")).collect();
assert!(run.len() > READ_BUFFER, "the run must reach the bound");
pump.out_writer.write_all(run.as_bytes()).await.unwrap();
wait_for_a_full_logs_channel(&pump.logs).await;
pump.flush().await;
let before = fs::metadata(&pump.out_path).unwrap().len();
let _ = pump.report_fds().await;
let drained = fs::metadata(&pump.out_path).unwrap().len() - before;
assert!(
drained > 0,
"the report must rescue what the reader was holding"
);
const RESCUED: usize = READ_BUFFER + RUN_LINE;
let ceiling = u64::try_from(RESCUED + RESCUED / RUN_LINE * LOG_STAMP_BYTES).unwrap();
assert!(
drained <= ceiling,
"the report drained {drained} bytes against a ceiling of {ceiling}, which is \
more than one bufferful: it is following the sheep rather than catching up"
);
}
#[derive(Clone, Debug, Default)]
struct WriteCounter {
writes: Arc<AtomicUsize>,
bytes: Arc<AtomicUsize>,
}
impl WriteCounter {
fn writes(&self) -> usize {
self.writes.load(Ordering::Relaxed)
}
fn bytes(&self) -> usize {
self.bytes.load(Ordering::Relaxed)
}
}
impl AsyncWrite for WriteCounter {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
self.writes.fetch_add(1, Ordering::Relaxed);
self.bytes.fetch_add(buf.len(), Ordering::Relaxed);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[tokio::test(start_paused = true)]
async fn a_run_of_lines_costs_one_write_per_bufferful_not_one_per_line() {
const LINE: &str = "012345678901234567890123456789012345678901234567890123456789012345678";
const LINE_BYTES: usize = LOG_STAMP_BYTES + LINE.len() + 1;
let lines = 3 * LOG_BUFFER / LINE_BYTES;
let sink = WriteCounter::default();
let mut log = LogFile {
record: record_lock(std::path::Path::new("test")),
path: PathBuf::from("counted.log"),
handle: Some(BufWriter::with_capacity(LOG_BUFFER, sink.clone())),
buffered_since: None,
stamp: String::new(),
};
for _ in 0..lines {
log.append(LINE).await;
}
log.flush().await.unwrap();
let total = lines * LINE_BYTES;
let ceiling = total.div_ceil(LOG_BUFFER) + 1; assert_eq!(
sink.bytes(),
total,
"buffering must not lose or repeat a byte"
);
assert!(
sink.writes() <= ceiling,
"{lines} lines cost {} writes; one per bufferful plus the closing flush is {ceiling}",
sink.writes()
);
assert!(
sink.writes() < lines,
"a write per line is the regression this exists to catch: \
{lines} lines, {} writes",
sink.writes()
);
}
#[tokio::test(start_paused = true)]
async fn a_line_from_a_sheep_that_then_goes_quiet_still_reaches_its_file() {
const LINE: &str = "the-only-line";
let sink = WriteCounter::default();
let mut log = LogFile {
record: record_lock(std::path::Path::new("test")),
path: PathBuf::from("quiet.log"),
handle: Some(BufWriter::with_capacity(LOG_BUFFER, sink.clone())),
buffered_since: None,
stamp: String::new(),
};
log.append(LINE).await;
assert_eq!(
sink.bytes(),
0,
"one short line must sit in the buffer, or there is nothing for the idle flush to do"
);
let armed = log
.buffered_since
.expect("an appended line must arm the flush deadline");
tokio::time::sleep_until(armed + IDLE_FLUSH).await;
log.flush().await.unwrap();
assert_eq!(
sink.bytes(),
LOG_STAMP_BYTES + LINE.len() + 1,
"the stamp, the line and its newline must reach the file once the window closes"
);
assert_eq!(
log.buffered_since, None,
"a flush must retire the deadline, or the pump re-arms it every window"
);
}
#[tokio::test]
async fn a_line_carries_the_time_it_was_written() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("stamped.log");
let mut log = LogFile::open(path.clone()).await;
log.append("the-sheep-said-this").await;
log.flush().await.expect("the file must be writable");
let written = fs::read_to_string(&path).unwrap();
let line = written.strip_suffix('\n').expect("one whole line");
let (stamp, rest) = line.split_at(LOG_STAMP_BYTES);
assert_eq!(
rest, "the-sheep-said-this",
"the sheep's own bytes must survive the prefix, whole and unaltered"
);
let stamp = stamp
.strip_suffix(' ')
.expect("one space must separate the stamp from the line");
let parsed = chrono::DateTime::parse_from_rfc3339(stamp)
.unwrap_or_else(|err| panic!("{stamp:?} must parse as RFC 3339: {err}"));
let drift = (chrono::Utc::now() - parsed.to_utc()).num_seconds().abs();
assert!(
drift < 60,
"{stamp:?} is {drift}s from now; the stamp must be the moment the line was written"
);
}
#[tokio::test(start_paused = true)]
async fn the_idle_flush_window_is_measured_from_the_oldest_buffered_line() {
let dir = tempfile::tempdir().unwrap();
let mut files = LogFiles {
out: LogFile::open(dir.path().join("out.log")).await,
err: LogFile::open(dir.path().join("err.log")).await,
pipes: PipeFds::default(),
#[cfg(unix)]
parked: false,
};
assert_eq!(
files.flush_deadline(),
None,
"an untouched pair owes no flush"
);
let oldest = Instant::now();
files.stream(false).append("out-first").await;
assert_eq!(files.flush_deadline(), Some(oldest + IDLE_FLUSH));
tokio::time::advance(IDLE_FLUSH / 2).await;
files.stream(false).append("out-second").await;
files.stream(true).append("err-first").await;
assert_eq!(
files.flush_deadline(),
Some(oldest + IDLE_FLUSH),
"a later line must not push the window out"
);
files.flush_idle().await;
assert_eq!(
files.flush_deadline(),
None,
"a flush must retire the deadline it satisfied"
);
}
#[tokio::test]
async fn a_rotation_lands_every_buffered_line_exactly_once() {
let mut pump = PumpHarness::start();
let mut before = String::new();
for n in 0..40 {
let line = format!("before-{n}");
pump.feed(false, &line).await;
before.push_str(&line);
before.push('\n');
}
assert!(
before.len() < LOG_BUFFER,
"the case needs lines small enough that the buffer may still hold them"
);
let rotated = pump.dir.path().join("out.log.1");
fs::rename(&pump.out_path, &rotated).unwrap();
pump.reopen().await;
assert_eq!(log_text(&rotated), before);
assert_eq!(log_text(&pump.out_path), "");
let mut after = String::new();
for n in 0..40 {
let line = format!("after-{n}");
pump.feed(false, &line).await;
after.push_str(&line);
after.push('\n');
}
pump.flush().await;
assert_eq!(log_text(&pump.out_path), after);
assert_eq!(
log_text(&rotated),
before,
"the archive must have stopped growing at the swap"
);
}
#[tokio::test]
async fn a_reopen_moves_both_streams_onto_the_recreated_paths() {
let mut pump = PumpHarness::start();
pump.feed(false, "before-out").await;
pump.feed(true, "before-err").await;
let rotated_out = pump.dir.path().join("out.log.1");
let rotated_err = pump.dir.path().join("err.log.1");
fs::rename(&pump.out_path, &rotated_out).unwrap();
fs::rename(&pump.err_path, &rotated_err).unwrap();
assert!(
!pump.out_path.exists(),
"sanity: the rename really moved it"
);
pump.reopen().await;
assert_eq!(log_text(&rotated_out), "before-out\n");
assert_eq!(log_text(&rotated_err), "before-err\n");
assert_eq!(log_text(&pump.out_path), "");
assert_eq!(log_text(&pump.err_path), "");
pump.feed(false, "after-out").await;
pump.feed(true, "after-err").await;
pump.reopen().await;
assert_eq!(log_text(&pump.out_path), "after-out\n");
assert_eq!(log_text(&pump.err_path), "after-err\n");
assert_eq!(log_text(&rotated_out), "before-out\n");
assert_eq!(log_text(&rotated_err), "before-err\n");
}
#[tokio::test]
async fn a_reopened_handle_still_appends_so_a_truncation_leaves_no_hole() {
let mut pump = PumpHarness::start();
pump.feed(false, "first").await;
fs::rename(&pump.out_path, pump.dir.path().join("out.log.1")).unwrap();
pump.reopen().await;
pump.feed(false, "second").await;
assert_file_settles(&pump.out_path, "second\n").await;
fs::File::create(&pump.out_path).unwrap();
assert_eq!(fs::metadata(&pump.out_path).unwrap().len(), 0);
pump.feed(false, "third").await;
assert_file_settles(&pump.out_path, "third\n").await;
}
#[cfg(unix)]
#[tokio::test]
async fn opening_a_symlinked_log_path_is_refused_rather_than_followed() {
let dir = tempfile::tempdir().unwrap();
let target = dir.path().join("someone-elses.conf");
let link = dir.path().join("web-out.log");
fs::write(&target, b"original").unwrap();
std::os::unix::fs::symlink(&target, &link).unwrap();
let error = open_append(&link)
.await
.expect_err("a symlinked log path must not be opened for appending");
assert_eq!(
fs::read(&target).unwrap(),
b"original",
"a refused open must not have reached the symlink's target at all"
);
assert_eq!(
error.to_string(),
crate::runner::SYMLINK_REFUSED,
"the operator who reads this needs the word symlink, not ELOOP's \
own wording about levels of them"
);
}
#[tokio::test]
async fn a_reopen_is_answered_while_both_streams_are_idle() {
let pump = PumpHarness::start();
pump.reopen().await;
fs::remove_file(&pump.out_path).unwrap();
fs::remove_file(&pump.err_path).unwrap();
pump.reopen().await;
assert!(pump.out_path.exists(), "stdout's path must be back");
assert!(pump.err_path.exists(), "stderr's path must be back");
}
#[tokio::test]
async fn a_reopen_is_answered_while_the_logs_channel_is_full() {
let mut pump = PumpHarness::start();
let flooded = CHANNEL_CAPACITY + 1;
let mut written = String::new();
for n in 0..flooded {
let line = format!("line-{n}\n");
pump.out_writer.write_all(line.as_bytes()).await.unwrap();
written.push_str(&line);
}
assert_file_settles(&pump.out_path, &written).await;
fs::remove_file(&pump.out_path).unwrap();
fs::remove_file(&pump.err_path).unwrap();
pump.reopen().await;
assert!(pump.out_path.exists(), "stdout's path must be back");
assert!(pump.err_path.exists(), "stderr's path must be back");
for n in 0..flooded {
let observed = timeout(PUMP_DEADLINE, pump.logs.recv())
.await
.expect("the pump must resume once `logs` has room")
.expect("the pump must not end while its streams are open");
assert_eq!(observed.line, format!("line-{n}"));
}
}
async fn let_the_pump_settle() {
for _ in 0..16 {
tokio::task::yield_now().await;
}
}
#[tokio::test]
async fn a_stream_that_has_ended_is_still_reopened() {
let mut pump = PumpHarness::start();
pump.feed(false, "before-out").await;
pump.feed(true, "before-err").await;
drop(pump.out_writer);
let_the_pump_settle().await;
fs::remove_file(&pump.out_path).unwrap();
fs::remove_file(&pump.err_path).unwrap();
let (done, ack) = oneshot::channel();
pump.ctl
.send(LogCtl::Reopen { done })
.await
.expect("a pump with one live stream still reads its control channel");
let outcome = timeout(PUMP_DEADLINE, ack)
.await
.expect("a reopen must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
assert_eq!(
outcome,
Ok(()),
"an ended stream's path is as openable as a live one's"
);
assert!(
pump.out_path.exists(),
"the ended stream's path must come back too"
);
assert!(
pump.err_path.exists(),
"the live stream's path must be back"
);
}
#[tokio::test]
async fn a_send_fails_once_the_pump_has_ended() {
let mut pump = PumpHarness::start();
pump.feed(false, "before-out").await;
pump.feed(true, "before-err").await;
drop(pump.out_writer);
drop(pump.err_writer);
let ended = timeout(PUMP_DEADLINE, pump.logs.recv())
.await
.expect("the pump must end once both streams reach EOF");
assert!(ended.is_none(), "a pump that has ended sends nothing more");
let (done, ack) = oneshot::channel();
assert!(
pump.ctl.send(LogCtl::Reopen { done }).await.is_err(),
"a reopen aimed at an ended pump must fail to send"
);
assert!(ack.await.is_err());
}
#[tokio::test]
async fn a_reopen_that_cannot_open_a_path_again_answers_with_the_failure() {
let pump = PumpHarness::start();
pump.reopen().await;
fs::rename(&pump.out_path, pump.dir.path().join("out.log.1")).unwrap();
fs::create_dir(&pump.out_path).unwrap();
fs::remove_file(&pump.err_path).unwrap();
let error = pump
.reopen_for_answer()
.await
.expect_err("a reopen that could not open stdout's path must say so");
assert!(
error.message.contains(pump.out_path.to_str().unwrap()),
"the failure must name the path it could not open: {error}"
);
assert!(
!error.message.contains(pump.err_path.to_str().unwrap()),
"stderr's path opened fine and must not be reported: {error}"
);
assert!(
pump.err_path.exists(),
"stderr must be reopened even though stdout's open failed"
);
}
#[tokio::test]
async fn a_flush_lands_both_streams_and_keeps_writing_afterwards() {
let mut pump = PumpHarness::start();
pump.feed(false, "out-before").await;
pump.feed(true, "err-before").await;
pump.flush().await;
assert_eq!(log_text(&pump.out_path), "out-before\n");
assert_eq!(log_text(&pump.err_path), "err-before\n");
pump.feed(false, "out-after").await;
assert_file_settles(&pump.out_path, "out-before\nout-after\n").await;
}
#[tokio::test]
async fn a_flush_reports_the_write_its_file_never_took() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out.log");
fs::write(&path, "").unwrap();
let mut log = LogFile {
record: record_lock(std::path::Path::new("test")),
path: path.clone(),
handle: Some(BufWriter::with_capacity(
LOG_BUFFER,
tokio::fs::File::open(&path).await.unwrap(),
)),
buffered_since: None,
stamp: String::new(),
};
log.append("never-lands").await;
let error = log
.flush()
.await
.expect_err("a write that never reached the file must fail the flush");
assert!(
error.message.starts_with(&format!("{}: ", path.display())),
"the failure must name the file it belongs to: {error}"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_log_file_from_an_open_handle_still_appends() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out.log");
fs::write(&path, "first\n").unwrap();
let handle = open_append(&path).await.unwrap();
let mut log = LogFile::from_file(path.clone(), handle);
log.append("second").await;
log.flush().await.expect("the carried handle must be live");
assert_eq!(
log_text(&path),
"first\nsecond\n",
"a carried handle must append, not write at its own offset"
);
}
#[tokio::test]
async fn dropping_the_logs_receiver_ends_a_pump_with_no_line_to_carry_the_news() {
let pump = PumpHarness::start();
pump.reopen().await;
drop(pump.logs);
timeout(PUMP_DEADLINE, pump.ctl.closed())
.await
.expect("a pump whose `logs` receiver is gone must end");
}
#[tokio::test]
async fn dropping_the_control_sender_ends_the_pump() {
let mut pump = PumpHarness::start();
pump.feed(false, "still-here").await;
drop(pump.ctl);
let after = timeout(PUMP_DEADLINE, pump.logs.recv())
.await
.expect("the pump must end once nothing can control it");
assert!(after.is_none(), "a pump that has ended sends nothing more");
}
const STDIN_BUFFER: usize = 4;
#[tokio::test]
async fn a_line_whose_caller_stopped_waiting_is_dropped_rather_than_written_later() {
use tokio::io::AsyncReadExt as _;
let (mut child_end, daemon_end) = tokio::io::duplex(STDIN_BUFFER);
let (to_stdin, rx) = mpsc::channel(CHANNEL_CAPACITY);
spawn_stdin_pump(Some(daemon_end), rx);
let (first_done, first_ack) = oneshot::channel();
to_stdin
.send(StdinWrite {
line: "AAAAAAAA".to_string(),
done: first_done,
})
.await
.unwrap();
let (second_done, second_ack) = oneshot::channel();
to_stdin
.send(StdinWrite {
line: "BBBB".to_string(),
done: second_done,
})
.await
.unwrap();
drop(second_ack);
let (third_done, third_ack) = oneshot::channel();
to_stdin
.send(StdinWrite {
line: "CCCC".to_string(),
done: third_done,
})
.await
.unwrap();
drop(to_stdin);
let mut written = Vec::new();
timeout(PUMP_DEADLINE, child_end.read_to_end(&mut written))
.await
.expect("the pump must drain and close the pipe")
.expect("reading the pipe must succeed");
assert_eq!(
String::from_utf8(written).unwrap(),
"AAAAAAAA\nCCCC\n",
"the abandoned line must not reach the app, and the live one must"
);
assert!(
timeout(PUMP_DEADLINE, first_ack)
.await
.expect("the first write must be acknowledged")
.expect("its sender must outlive the write")
.is_ok()
);
assert!(
timeout(PUMP_DEADLINE, third_ack)
.await
.expect("the third write must be acknowledged")
.expect("its sender must outlive the write")
.is_ok()
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_report_names_the_stdin_write_end_it_was_told_about() {
let dir = tempfile::tempdir().unwrap();
let (_child_end, daemon_end) = std::io::pipe().unwrap();
let (_out_writer, out_reader) = tokio::io::duplex(STREAM_BUFFER);
let (logs_tx, _logs) = mpsc::channel(CHANNEL_CAPACITY);
let (ctl, ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
let pipes = PipeFds {
out: None,
err: None,
stdin: Some(daemon_end.as_raw_fd()),
channel: None,
};
spawn_log_pump(
Some(out_reader),
None::<DuplexStream>,
LogSink::Path(dir.path().join("out.log")),
LogSink::Path(dir.path().join("err.log")),
logs_tx,
ctl_rx,
pipes,
);
let (done, ack) = oneshot::channel();
ctl.send(LogCtl::ReportFds { done }).await.unwrap();
let fds = timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
assert_eq!(fds.stdin, Some(daemon_end.as_raw_fd()));
}
#[cfg(unix)]
#[tokio::test]
async fn an_adopted_stdin_pipe_carries_a_line_to_the_child() {
let dir = tempfile::tempdir().unwrap();
let (mut child_end, daemon_end) = std::io::pipe().unwrap();
let daemon_end = tokio::net::unix::pipe::Sender::from_file(std::fs::File::from(
std::os::fd::OwnedFd::from(daemon_end),
))
.expect("the daemon's end of a stdin pipe is writable");
let (_proc, io) = TokioRunner::new()
.adopt(adopt_spec(&dir, Some(daemon_end), None))
.expect("the real runner must be able to adopt");
let (done, ack) = oneshot::channel();
io.to_stdin
.send(StdinWrite {
line: "whisper".to_string(),
done,
})
.await
.expect("an adopted sheep must still have a stdin pump");
timeout(PUMP_DEADLINE, ack)
.await
.expect("the write must be acknowledged")
.expect("the pump must outlive the write")
.expect("the write must succeed");
let mut buf = [0_u8; 8];
std::io::Read::read_exact(&mut child_end, &mut buf).expect("the child end must read");
assert_eq!(&buf, b"whisper\n");
}
#[cfg(unix)]
#[tokio::test]
async fn an_adopted_sheep_without_a_stdin_pipe_has_a_closed_channel() {
let dir = tempfile::tempdir().unwrap();
let (_proc, io) = TokioRunner::new()
.adopt(adopt_spec(&dir, None, None))
.expect("the real runner must be able to adopt");
assert!(io.to_stdin.is_closed());
}
#[cfg(unix)]
#[tokio::test]
async fn a_report_names_the_shepherd_channel_it_was_told_about() {
let dir = tempfile::tempdir().unwrap();
let (daemon_end, _child_end) = std::os::unix::net::UnixStream::pair().unwrap();
let (_out_writer, out_reader) = tokio::io::duplex(STREAM_BUFFER);
let (logs_tx, _logs) = mpsc::channel(CHANNEL_CAPACITY);
let (ctl, ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
let pipes = PipeFds {
out: None,
err: None,
stdin: None,
channel: Some(daemon_end.as_raw_fd()),
};
spawn_log_pump(
Some(out_reader),
None::<DuplexStream>,
LogSink::Path(dir.path().join("out.log")),
LogSink::Path(dir.path().join("err.log")),
logs_tx,
ctl_rx,
pipes,
);
let (done, ack) = oneshot::channel();
ctl.send(LogCtl::ReportFds { done }).await.unwrap();
let fds = timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
assert_eq!(fds.channel, Some(daemon_end.as_raw_fd()));
}
#[cfg(unix)]
#[tokio::test]
async fn an_adopted_shepherd_channel_carries_both_directions() {
use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _};
let dir = tempfile::tempdir().unwrap();
let (daemon_end, child_end) = tokio::net::UnixStream::pair().unwrap();
let (child_read, mut child_write) = tokio::io::split(child_end);
let mut child = tokio::io::BufReader::new(child_read);
let (_proc, mut io) = TokioRunner::new()
.adopt(adopt_spec(&dir, None, Some(daemon_end)))
.expect("the real runner must be able to adopt");
io.to_child
.send(ShepherdMessage::Shutdown)
.await
.expect("an adopted sheep must still have a channel writer");
let mut line = String::new();
timeout(PUMP_DEADLINE, child.read_line(&mut line))
.await
.expect("the shepherd's message must reach the child's end")
.expect("the child end must read");
assert_eq!(line.trim_end(), r#"{"kind":"shutdown"}"#);
child_write
.write_all(b"{\"kind\":\"ready\"}\n")
.await
.unwrap();
let back = timeout(PUMP_DEADLINE, io.from_child.recv())
.await
.expect("an adopted sheep must still have a channel reader")
.expect("the reader must forward what the child said");
assert_eq!(back, ChildMessage::Ready);
}
#[cfg(unix)]
#[tokio::test]
async fn an_adopted_sheep_without_a_channel_has_closed_channel_ends() {
let dir = tempfile::tempdir().unwrap();
let (_proc, mut io) = TokioRunner::new()
.adopt(adopt_spec(&dir, None, None))
.expect("the real runner must be able to adopt");
assert!(io.to_child.is_closed());
assert!(
io.from_child.recv().await.is_none(),
"a sheep with no channel must report one that is over, not one that is quiet"
);
}
#[cfg(unix)]
fn adopt_spec(
dir: &tempfile::TempDir,
stdin_pipe: Option<tokio::net::unix::pipe::Sender>,
channel: Option<tokio::net::UnixStream>,
) -> AdoptSpec {
AdoptSpec {
pid: std::process::id(),
out_file: dir.path().join("out.log"),
err_file: dir.path().join("err.log"),
out_pipe: None,
err_pipe: None,
out_log: None,
err_log: None,
stdin_pipe,
channel,
reaper: Arc::new(crate::runner::AdoptedReaper::new()),
}
}
#[cfg(unix)]
fn child_spec(dir: &tempfile::TempDir, program: &str, args: &[&str], stdin: bool) -> SpawnSpec {
SpawnSpec {
name: "web".to_string(),
program: program.to_string(),
args: args.iter().map(|arg| (*arg).to_string()).collect(),
cwd: None,
env: BTreeMap::new(),
out_file: dir.path().join("out.log"),
err_file: dir.path().join("err.log"),
channel: false,
stdin,
credentials: None,
}
}
#[cfg(unix)]
#[tokio::test]
async fn a_spawn_reports_the_write_end_it_put_on_the_childs_stdin() {
use nix::fcntl::{FcntlArg, OFlag, fcntl};
let dir = tempfile::tempdir().unwrap();
let (_proc, io) = TokioRunner::new()
.spawn(&child_spec(&dir, "/bin/cat", &[], true))
.unwrap();
let (done, ack) = oneshot::channel();
io.log_ctl.send(LogCtl::ReportFds { done }).await.unwrap();
let fds = timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
let stdin = fds
.stdin
.expect("a sheep spawned with a stdin pipe carries its write end");
let flags = OFlag::from_bits_truncate(fcntl(stdin, FcntlArg::F_GETFL).unwrap());
assert_eq!(
flags & OFlag::O_ACCMODE,
OFlag::O_WRONLY,
"the number reported for stdin must be the end the daemon writes"
);
assert_ne!(Some(stdin), fds.out_pipe);
assert_ne!(Some(stdin), fds.err_pipe);
drop(io);
}
#[cfg(unix)]
#[tokio::test]
async fn a_spawn_without_stdin_reports_no_descriptor_for_it() {
let dir = tempfile::tempdir().unwrap();
let (mut proc, io) = TokioRunner::new()
.spawn(&child_spec(&dir, "/bin/sleep", &["30"], false))
.unwrap();
let (done, ack) = oneshot::channel();
io.log_ctl.send(LogCtl::ReportFds { done }).await.unwrap();
let fds = timeout(PUMP_DEADLINE, ack)
.await
.expect("a descriptor report must be acknowledged")
.expect("the pump must answer rather than drop the acknowledgement");
assert_eq!(fds.stdin, None);
proc.signal_process(OperatorSignal::Kill).unwrap();
drop(io);
}
fn preflight_spec(program: &str, cwd: Option<PathBuf>, path: Option<&str>) -> SpawnSpec {
let mut env = BTreeMap::new();
if let Some(path) = path {
env.insert("PATH".to_string(), path.to_string());
}
SpawnSpec {
name: "web".to_string(),
program: program.to_string(),
args: Vec::new(),
cwd,
env,
out_file: PathBuf::from("/dev/null"),
err_file: PathBuf::from("/dev/null"),
channel: false,
stdin: false,
credentials: None,
}
}
#[test]
fn a_program_that_is_really_there_is_nothing_to_report() {
let dir = tempfile::tempdir().unwrap();
let bin = dir.path().join("srv");
fs::write(&bin, "#!/bin/sh\n").unwrap();
let path_dir = dir.path().to_string_lossy().into_owned();
for spec in [
preflight_spec(&bin.to_string_lossy(), None, None),
preflight_spec("./srv", Some(dir.path().to_path_buf()), None),
preflight_spec("srv", None, Some(&path_dir)),
preflight_spec("srv", None, Some(&format!("/nonexistent:{path_dir}"))),
preflight_spec("./srv", None, None),
preflight_spec("srv", None, None),
preflight_spec("srv", None, Some("")),
preflight_spec("", None, None),
] {
assert_eq!(
what_exec_will_find(&spec),
Preflight::Unknown,
"reported something about a spec it cannot be certain about: {spec:?}"
);
}
}
#[test]
fn an_absent_path_is_impossible_and_names_the_path_that_was_tried() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(
what_exec_will_find(&preflight_spec(
"./proto-enum-api",
Some(dir.path().to_path_buf()),
None,
)),
Preflight::Impossible(format!(
"no such file: {}",
dir.path().join("./proto-enum-api").display()
)),
);
let absent = if cfg!(windows) {
r"C:\nonexistent\srv"
} else {
"/nonexistent/srv"
};
assert_eq!(
what_exec_will_find(&preflight_spec(absent, None, None)),
Preflight::Impossible(format!("no such file: {absent}")),
);
}
#[test]
fn a_filesystem_error_that_is_not_absence_is_never_impossible() {
let dir = tempfile::tempdir().unwrap();
#[cfg(unix)]
let unreadable = {
let wall = dir.path().join("wall");
fs::write(&wall, "not a directory").unwrap();
wall.join("srv")
};
#[cfg(windows)]
let unreadable = dir.path().join("no<such<name");
let kind = fs::metadata(&unreadable).unwrap_err().kind();
assert_ne!(
kind,
io::ErrorKind::NotFound,
"the provocation must be an error OTHER than not-found, or this \
case proves nothing: {kind:?}"
);
assert_eq!(
what_exec_will_find(&preflight_spec(&unreadable.to_string_lossy(), None, None)),
Preflight::Unknown,
"a path shep could not read is a suspicion, not a certainty, and \
must never refuse a batch"
);
#[cfg(unix)]
{
if nix::unistd::Uid::effective().is_root() {
return;
}
let locked = dir.path().join("locked");
fs::create_dir(&locked).unwrap();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap();
let behind_the_wall = locked.join("srv");
let observed = fs::metadata(&behind_the_wall)
.map(|_| ())
.map_err(|e| e.kind());
let verdict = what_exec_will_find(&preflight_spec(
&behind_the_wall.to_string_lossy(),
None,
None,
));
fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap();
assert_eq!(
observed,
Err(io::ErrorKind::PermissionDenied),
"the chmod must actually bite, or the assertion below is vacuous"
);
assert_eq!(
verdict,
Preflight::Unknown,
"a permission error on the way to the program must not take the rest \
of the flock down with it"
);
}
}
#[test]
fn a_bare_command_off_the_path_is_only_ever_doubtful() {
let found = what_exec_will_find(&preflight_spec("node", None, Some("/nonexistent")));
assert_eq!(
found,
Preflight::Doubtful("`node` is not on the shepherd's PATH (/nonexistent)".to_string()),
);
assert!(
!matches!(found, Preflight::Impossible(_)),
"a claim about an environment must never refuse a batch"
);
}
#[test]
fn a_long_path_is_summarised_and_a_startup_units_own_path_is_not() {
let sep = PATH_LIST_SEPARATOR;
let join = |entries: &[&str]| entries.join(&sep.to_string());
let fallback = join(&["/usr/local/bin", "/usr/bin", "/bin"]);
assert_eq!(
summarise_path(&fallback),
fallback,
"the PATH a unit actually gets must print in full"
);
let long = join(&["/a", "/b", "/c", "/d", "/e", "/f"]);
assert_eq!(
summarise_path(&long),
format!("{} and 2 more entries", join(&["/a", "/b", "/c", "/d"]))
);
let capped = join(&["/a", "/b", "/c", "/d"]);
assert_eq!(summarise_path(&capped), capped);
}
#[cfg(unix)]
#[test]
fn a_zero_pid_is_refused_before_it_can_reach_the_daemons_own_group() {
let err = signal_group(0, Signal::SIGCONT).unwrap_err();
assert_eq!(
err.to_string(),
"signal delivery failed: pid 0 is not a signallable process id"
);
}
}