#[cfg(unix)]
use std::os::fd::{AsRawFd, OwnedFd, RawFd};
#[cfg(unix)]
use std::os::unix::process::ExitStatusExt as _;
use std::process::Stdio;
#[cfg(unix)]
use std::sync::Arc;
use std::sync::{Mutex, PoisonError};
#[cfg(unix)]
use command_fds::{CommandFdExt, FdMapping};
#[cfg(unix)]
use nix::sys::signal::{self, Signal};
#[cfg(unix)]
use nix::unistd::Pid;
use shep_core::signals::OperatorSignal;
#[cfg(unix)]
use tokio::net::UnixStream;
use tokio::process::{Child, Command};
use tokio::sync::mpsc;
use crate::channel::CHANNEL_VERSION;
#[cfg(unix)]
use crate::runner::{AdoptSpec, AdoptedReaper};
use crate::runner::{
DogLink, ExitOutcome, Preflight, ProcIo, ProcessRunner, RunnerError, RunningProcess, SpawnSpec,
StopSignal,
};
#[cfg(windows)]
use shep_core::dogs::DOG_LINK_PIPE_VAR;
#[cfg(unix)]
use shep_core::dogs::{DOG_LINK_FD, DOG_LINK_FD_VAR};
use super::CHANNEL_CAPACITY;
#[cfg(unix)]
use super::dog_link;
#[cfg(unix)]
use super::log_file::carried_sink;
use super::log_file::{LogSink, PipeFds};
use super::preflight::what_exec_will_find;
#[cfg(unix)]
use super::pty::{self, Stdout};
use super::pump::{spawn_channel_pumps, spawn_log_pump, spawn_stdin_pump};
#[cfg(windows)]
use super::windows_pipe;
#[derive(Debug)]
pub struct TokioRunner {
links: mpsc::UnboundedSender<DogLink>,
links_rx: Mutex<Option<mpsc::UnboundedReceiver<DogLink>>>,
}
impl Default for TokioRunner {
fn default() -> Self {
Self::new()
}
}
#[cfg(windows)]
const KILL_TREE_EXIT_CODE: u32 = 137;
impl TokioRunner {
#[must_use]
pub fn new() -> Self {
let (links, links_rx) = mpsc::unbounded_channel();
Self {
links,
links_rx: Mutex::new(Some(links_rx)),
}
}
}
#[derive(Debug)]
pub struct TokioProc {
pid: u32,
proc: Supervised,
#[cfg(unix)]
leader_only_stop: bool,
#[cfg(windows)]
job: shep_tree::job::Job,
}
#[derive(Debug)]
enum Supervised {
Spawned(Child),
#[cfg(unix)]
Adopted(Arc<AdoptedReaper>),
}
impl RunningProcess for TokioProc {
fn pid(&self) -> u32 {
self.pid
}
async fn wait(&mut self) -> ExitOutcome {
let child = match &mut self.proc {
Supervised::Spawned(child) => child,
#[cfg(unix)]
Supervised::Adopted(reaper) => {
return match reaper.wait(self.pid).await {
Ok(outcome) => outcome,
Err(error) => {
tracing::error!(pid = self.pid, %error, "adopted process wait failed");
ExitOutcome {
code: None,
signal: None,
}
}
};
}
};
match child.wait().await {
Ok(status) => ExitOutcome {
code: status.code(),
#[cfg(windows)]
signal: None,
#[cfg(unix)]
signal: status.signal(),
},
Err(error) => {
tracing::error!(pid = self.pid, %error, "process wait() failed");
ExitOutcome {
code: None,
signal: None,
}
}
}
}
#[cfg(unix)]
fn signal(&mut self, sig: StopSignal) -> Result<(), RunnerError> {
if self.leader_only_stop {
return signal_leader(self.pid, to_nix_signal(sig));
}
signal_group(self.pid, to_nix_signal(sig))
}
#[cfg(windows)]
fn signal(&mut self, sig: StopSignal) -> Result<(), RunnerError> {
Err(RunnerError::SignalFailed(format!(
"Windows cannot deliver {sig:?} to another process; \
an app that needs a graceful stop must opt into the shepherd \
channel (shutdown_with_message), otherwise the stop escalates \
to a forced termination after kill_timeout"
)))
}
#[cfg(unix)]
fn kill_tree(&mut self) -> Result<(), RunnerError> {
signal_group(self.pid, Signal::SIGKILL)
}
#[cfg(windows)]
fn kill_tree(&mut self) -> Result<(), RunnerError> {
self.job
.terminate(KILL_TREE_EXIT_CODE)
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
#[cfg(windows)]
fn signal_process(&mut self, sig: OperatorSignal) -> Result<(), RunnerError> {
let Supervised::Spawned(child) = &mut self.proc;
match sig {
OperatorSignal::Kill => child
.start_kill()
.map_err(|error| RunnerError::SignalFailed(error.to_string())),
other => Err(refused_on_windows(other)),
}
}
#[cfg(unix)]
fn signal_process(&mut self, sig: OperatorSignal) -> Result<(), RunnerError> {
signal_pid(self.pid, sig)
}
}
#[cfg(unix)]
pub(crate) fn signal_pid(pid: u32, sig: OperatorSignal) -> Result<(), RunnerError> {
let pid = signallable_pid(pid)?;
signal::kill(Pid::from_raw(pid), to_nix_operator_signal(sig))
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
#[cfg(windows)]
pub(crate) fn signal_pid(pid: u32, sig: OperatorSignal) -> Result<(), RunnerError> {
match sig {
OperatorSignal::Kill => crate::sys_windows::terminate_process(pid, KILL_TREE_EXIT_CODE)
.map_err(|error| RunnerError::SignalFailed(error.to_string())),
other => Err(refused_on_windows(other)),
}
}
#[cfg(windows)]
fn refused_on_windows(sig: OperatorSignal) -> RunnerError {
RunnerError::SignalFailed(format!(
"Windows has no way to deliver {sig:?} to another process; only SIGKILL is available here"
))
}
#[cfg(unix)]
pub(super) fn signallable_pid(pid: u32) -> Result<i32, RunnerError> {
i32::try_from(pid)
.ok()
.filter(|pid| *pid > 0)
.ok_or_else(|| {
RunnerError::SignalFailed(format!("pid {pid} is not a signallable process id"))
})
}
#[cfg(unix)]
pub(crate) fn signal_group(pid: u32, sig: Signal) -> Result<(), RunnerError> {
signallable_pid(pid)?;
shep_tree::group::signal_group(pid, to_tree_signal(sig)?)
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
#[cfg(unix)]
fn signal_leader(pid: u32, sig: Signal) -> Result<(), RunnerError> {
let pid = signallable_pid(pid)?;
signal::kill(Pid::from_raw(pid), sig)
.map_err(|error| RunnerError::SignalFailed(error.to_string()))
}
#[cfg(unix)]
fn to_tree_signal(sig: Signal) -> Result<shep_tree::Signal, RunnerError> {
match sig {
Signal::SIGTERM => Ok(shep_tree::Signal::Term),
Signal::SIGINT => Ok(shep_tree::Signal::Int),
Signal::SIGQUIT => Ok(shep_tree::Signal::Quit),
Signal::SIGUSR2 => Ok(shep_tree::Signal::Usr2),
Signal::SIGKILL => Ok(shep_tree::Signal::Kill),
other => Err(RunnerError::SignalFailed(format!(
"{other} cannot be sent to a process group"
))),
}
}
#[cfg(unix)]
fn to_nix_signal(sig: StopSignal) -> Signal {
match sig {
StopSignal::Term => Signal::SIGTERM,
StopSignal::Int => Signal::SIGINT,
StopSignal::Quit => Signal::SIGQUIT,
StopSignal::Usr2 => Signal::SIGUSR2,
StopSignal::Kill => Signal::SIGKILL,
}
}
#[cfg(unix)]
fn to_nix_operator_signal(sig: OperatorSignal) -> Signal {
match sig {
OperatorSignal::Hup => Signal::SIGHUP,
OperatorSignal::Int => Signal::SIGINT,
OperatorSignal::Quit => Signal::SIGQUIT,
OperatorSignal::Term => Signal::SIGTERM,
OperatorSignal::Usr1 => Signal::SIGUSR1,
OperatorSignal::Usr2 => Signal::SIGUSR2,
OperatorSignal::Winch => Signal::SIGWINCH,
OperatorSignal::Cont => Signal::SIGCONT,
OperatorSignal::Kill => Signal::SIGKILL,
}
}
impl ProcessRunner for TokioRunner {
type Proc = TokioProc;
fn preflight(&self, spec: &SpawnSpec) -> Preflight {
what_exec_will_find(spec)
}
#[cfg(unix)]
fn adopt(&self, spec: AdoptSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
let AdoptSpec {
name,
pid,
out_file,
err_file,
out_pipe,
err_pipe,
out_log,
err_log,
log_timestamps,
leader_only_stop,
stdin_pipe,
channel,
dog_link,
dog_topics,
launch_dog,
reaper,
pty,
} = spec;
let (logs_tx, logs_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (log_ctl_tx, log_ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
let pty_master = pty
.map(pty::PtyMaster::from_file)
.transpose()
.map_err(|error| RunnerError::AdoptFailed(format!("pty master: {error}")))?;
let pipes = PipeFds {
out: out_pipe.as_ref().map(AsRawFd::as_raw_fd),
err: err_pipe.as_ref().map(AsRawFd::as_raw_fd),
stdin: stdin_pipe.as_ref().map(AsRawFd::as_raw_fd),
channel: channel.as_ref().map(AsRawFd::as_raw_fd),
pty: pty_master.as_ref().map(pty::PtyMaster::raw_fd),
};
let (to_stdin, to_stdin_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (stdout, pty_handle, to_stdin_rx) = match pty_master {
Some(master) => {
let (stdout, handle) = pty::wire(master, to_stdin_rx, pid);
(Some(stdout), Some(handle), None)
}
None => (out_pipe.map(Stdout::Carried), None, Some(to_stdin_rx)),
};
spawn_log_pump(
stdout,
err_pipe,
carried_sink(out_file, out_log),
carried_sink(err_file, err_log),
log_timestamps,
logs_tx,
log_ctl_rx,
pipes,
);
if let Some(stream) = dog_link {
let _ = self.links.send(match launch_dog {
Some(dog) => DogLink {
name: dog,
launch_for: Some(name),
pid,
stream,
resumed: dog_topics,
},
None => DogLink {
name,
launch_for: None,
pid,
stream,
resumed: dog_topics,
},
});
}
let (from_child_tx, from_child) = mpsc::channel(CHANNEL_CAPACITY);
let (to_child, to_child_rx) = mpsc::channel(CHANNEL_CAPACITY);
if let Some(channel) = channel {
spawn_channel_pumps(channel, from_child_tx, to_child_rx);
} else {
drop(from_child_tx);
drop(to_child_rx);
}
if let Some(to_stdin_rx) = to_stdin_rx
&& let Some(stdin_pipe) = stdin_pipe
{
spawn_stdin_pump(Some(stdin_pipe), b'\n', to_stdin_rx);
}
Ok((
TokioProc {
pid,
proc: Supervised::Adopted(reaper),
leader_only_stop,
},
ProcIo {
logs: logs_rx,
from_child,
to_child,
log_ctl: log_ctl_tx,
to_stdin,
pty: pty_handle,
},
))
}
fn dog_links(&self) -> Option<mpsc::UnboundedReceiver<DogLink>> {
self.links_rx
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take()
}
fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
let mut command = Command::new(&spec.program);
command.args(&spec.args);
if let Some(cwd) = &spec.cwd {
command.current_dir(cwd);
}
let mut env = spec.env.clone();
#[cfg(unix)]
let pty_master =
if spec.pty {
Some(pty::attach(&mut command).map_err(|error| {
RunnerError::SpawnFailed(format!("pseudo-terminal: {error}"))
})?)
} else {
None
};
#[cfg(unix)]
let piped = pty_master.is_none();
#[cfg(windows)]
let piped = true;
if piped {
command.stdin(if spec.stdin {
Stdio::piped()
} else {
Stdio::null()
});
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
#[cfg(unix)]
command.process_group(0);
}
#[cfg(windows)]
{
const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200;
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
command.creation_flags(CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW);
}
#[cfg(unix)]
if let Some(creds) = spec.credentials {
if let Some(gid) = creds.gid {
command.gid(gid);
}
command.uid(creds.uid);
}
#[cfg(windows)]
debug_assert!(
spec.credentials.is_none(),
"privilege::resolve must refuse user/group on Windows before a spawn is reached"
);
let (from_child_tx, from_child_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (to_child_tx, to_child_rx) = mpsc::channel(CHANNEL_CAPACITY);
#[cfg(windows)]
if spec.channel {
let (pipe, mut listener) = windows_pipe::bind("channel", "shepherd channel pipe")?;
env.insert("SHEP_CHANNEL_PIPE".to_string(), pipe.display().to_string());
env.insert(
"SHEP_CHANNEL_VERSION".to_string(),
CHANNEL_VERSION.to_string(),
);
let from_child_tx = from_child_tx.clone();
tokio::spawn(async move {
let watcher = from_child_tx.clone();
tokio::select! {
accepted = listener.accept() => match accepted {
Ok(daemon_end) => {
spawn_channel_pumps(daemon_end, from_child_tx, to_child_rx);
}
Err(error) => {
tracing::warn!(%error, "shepherd channel accept failed");
}
},
() = watcher.closed() => {}
}
});
} else {
drop(to_child_rx);
}
#[cfg(windows)]
let mut dog_pipe = None;
#[cfg(windows)]
if spec.dog_link {
let (pipe, listener) = windows_pipe::bind("dog", "dog connection pipe")?;
env.insert(DOG_LINK_PIPE_VAR.to_string(), pipe.display().to_string());
dog_pipe = Some(listener);
}
#[cfg(unix)]
let mut channel_fd: Option<RawFd> = None;
#[cfg(unix)]
let mut mappings: Vec<FdMapping> = Vec::new();
#[cfg(unix)]
let mut channel_child_fd: RawFd = 3;
#[cfg(unix)]
let listening = match &spec.listen {
Some(sockets) if !sockets.is_empty() => Some(sockets),
_ => None,
};
#[cfg(unix)]
if let Some(sockets) = listening {
if spec.dog_link {
return Err(RunnerError::SpawnFailed(
"a dog's connection and listen sockets both claim fd 4".to_string(),
));
}
let duplicates = sockets.duplicates().map_err(|error| {
RunnerError::SpawnFailed(format!("listen socket duplicate: {error}"))
})?;
for parent_fd in duplicates {
mappings.push(FdMapping {
parent_fd,
child_fd: channel_child_fd,
});
channel_child_fd += 1;
}
env.insert("LISTEN_FDS".to_string(), sockets.len().to_string());
env.insert(
"LISTEN_FDNAMES".to_string(),
sockets.names().collect::<Vec<_>>().join(":"),
);
}
#[cfg(unix)]
if spec.channel {
env.insert("SHEP_CHANNEL_FD".to_string(), channel_child_fd.to_string());
env.insert(
"SHEP_CHANNEL_VERSION".to_string(),
CHANNEL_VERSION.to_string(),
);
let (daemon_end, child_end) = UnixStream::pair().map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel socketpair: {error}"))
})?;
let std_child_end = child_end.into_std().map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel into_std: {error}"))
})?;
std_child_end.set_nonblocking(false).map_err(|error| {
RunnerError::SpawnFailed(format!("shepherd channel set_nonblocking: {error}"))
})?;
let child_fd = OwnedFd::from(std_child_end);
mappings.push(FdMapping {
parent_fd: child_fd,
child_fd: channel_child_fd,
});
channel_fd = Some(daemon_end.as_raw_fd());
spawn_channel_pumps(daemon_end, from_child_tx, to_child_rx);
} else {
drop(from_child_tx);
drop(to_child_rx);
}
#[cfg(unix)]
let mut dog_end = None;
#[cfg(unix)]
let launch_dog = spec
.launch_link
.as_deref()
.filter(|_| !spec.dog_link && listening.is_none());
#[cfg(unix)]
if spec.dog_link || launch_dog.is_some() {
let (daemon_end, mapping) = dog_link::pair()?;
env.insert(DOG_LINK_FD_VAR.to_string(), DOG_LINK_FD.to_string());
mappings.push(mapping);
dog_end = Some(daemon_end);
}
#[cfg(target_os = "linux")]
if let Some(dir) = &spec.cgroup {
crate::cgroup::join_on_exec(command.as_std_mut(), dir, spec.into())
.map_err(|error| crate::cgroup::spawn_failed(&spec.name, dir, &error))?;
}
#[cfg(unix)]
if !mappings.is_empty() {
command
.as_std_mut()
.fd_mappings(mappings)
.map_err(|error| {
#[cfg(target_os = "linux")]
crate::cgroup::release(spec.cgroup.as_deref());
RunnerError::SpawnFailed(format!("fd mapping: {error}"))
})?;
}
#[cfg(unix)]
if listening.is_some() {
let block = crate::sys::ListenEnv::new(&env).map_err(|error| {
#[cfg(target_os = "linux")]
crate::cgroup::release(spec.cgroup.as_deref());
RunnerError::SpawnFailed(format!("environment: {error}"))
})?;
crate::sys::set_listen_env(command.as_std_mut(), block);
} else {
command.env_clear();
command.envs(&env);
}
#[cfg(windows)]
{
command.env_clear();
command.envs(&env);
}
let mut child = command.spawn().map_err(|error| {
#[cfg(target_os = "linux")]
if let Some(dir) = &spec.cgroup {
return crate::cgroup::spawn_failed(&spec.name, dir, &error);
}
RunnerError::SpawnFailed(error.to_string())
})?;
drop(command);
let pid = child.id().ok_or_else(|| {
RunnerError::SpawnFailed("child exited before its pid could be read".to_string())
})?;
#[cfg(unix)]
if let Some(stream) = dog_end {
let _ = self.links.send(match launch_dog {
Some(dog) => DogLink {
name: dog.to_string(),
launch_for: Some(spec.name.clone()),
pid,
stream,
resumed: None,
},
None => DogLink {
name: spec.name.clone(),
launch_for: None,
pid,
stream,
resumed: None,
},
});
}
#[cfg(windows)]
let job = {
let job = shep_tree::job::Job::create().map_err(|error| {
RunnerError::SpawnFailed(format!("job object for {}: {error}", spec.name))
})?;
let handle = child.raw_handle().ok_or_else(|| {
RunnerError::SpawnFailed("child exited before it could be contained".to_string())
})?;
if let Err(error) = job.assign(handle) {
let _ = child.start_kill();
return Err(RunnerError::SpawnFailed(format!(
"could not contain {} in a job object: {error}",
spec.name
)));
}
job
};
let (logs_tx, logs_rx) = mpsc::channel(CHANNEL_CAPACITY);
let (log_ctl_tx, log_ctl_rx) = mpsc::channel(CHANNEL_CAPACITY);
#[cfg(windows)]
if let Some(listener) = dog_pipe {
windows_pipe::serve_dog_link(
listener,
spec.name.clone(),
pid,
self.links.clone(),
log_ctl_tx.clone(),
);
}
#[cfg(unix)]
let pipes = PipeFds {
out: child.stdout.as_ref().map(AsRawFd::as_raw_fd),
err: child.stderr.as_ref().map(AsRawFd::as_raw_fd),
stdin: child.stdin.as_ref().map(AsRawFd::as_raw_fd),
channel: channel_fd,
pty: pty_master.as_ref().map(pty::PtyMaster::raw_fd),
};
#[cfg(not(unix))]
let pipes = PipeFds;
let (to_stdin_tx, to_stdin_rx) = mpsc::channel(CHANNEL_CAPACITY);
#[cfg(unix)]
let (stdout, pty, to_stdin_rx) = match pty_master {
Some(master) => {
let (stdout, handle) = pty::wire(master, to_stdin_rx, pid);
(Some(stdout), Some(handle), None)
}
None => (
child.stdout.take().map(Stdout::Pipe),
None,
Some(to_stdin_rx),
),
};
#[cfg(windows)]
let (stdout, pty, to_stdin_rx) = (child.stdout.take(), None, Some(to_stdin_rx));
spawn_log_pump(
stdout,
child.stderr.take(),
LogSink::Path(spec.out_file.clone()),
LogSink::Path(spec.err_file.clone()),
spec.log_timestamps,
logs_tx,
log_ctl_rx,
pipes,
);
if let Some(to_stdin_rx) = to_stdin_rx {
if spec.stdin {
spawn_stdin_pump(child.stdin.take(), b'\n', to_stdin_rx);
} else {
drop(to_stdin_rx);
}
}
let io = ProcIo {
logs: logs_rx,
from_child: from_child_rx,
to_child: to_child_tx,
log_ctl: log_ctl_tx,
to_stdin: to_stdin_tx,
pty,
};
Ok((
TokioProc {
pid,
proc: Supervised::Spawned(child),
#[cfg(unix)]
leader_only_stop: spec.leader_only_stop,
#[cfg(windows)]
job,
},
io,
))
}
}