use std::{
collections::VecDeque,
env,
ffi::OsStr,
fmt,
fs::remove_file,
io::{self, ErrorKind, IoSliceMut},
mem::MaybeUninit,
os::{
fd::{AsFd, AsRawFd, OwnedFd, RawFd},
unix::{net::UnixDatagram, process::CommandExt},
},
path::{Path, PathBuf},
process::{self, Command},
sync::{
Arc, Condvar, Mutex, MutexGuard, PoisonError,
atomic::{AtomicUsize, Ordering},
},
thread::{self, JoinHandle},
time::{Duration, Instant},
};
use rustix::{
cmsg_space,
io::fcntl_dupfd_cloexec,
net::{RecvAncillaryBuffer, RecvAncillaryMessage, RecvFlags, recvmsg},
};
use crate::{
Error, Passed, Result, Store,
notifier::{MAX_FDS, Notifier},
passed::START,
queue::{self, Queue},
};
pub struct Manager {
path: PathBuf,
shared: Arc<Shared>,
serving: Option<JoinHandle<()>>,
activation: Vec<(String, OwnedFd)>,
}
impl Manager {
pub fn new() -> Result<Self> {
let path = env::temp_dir().join(unique_name());
let socket = UnixDatagram::bind(&path)?;
let shared = Arc::new(Shared {
state: Mutex::new(State {
store_max: DEFAULT_STORE_MAX,
..State::default()
}),
changed: Condvar::new(),
});
let serving = {
let shared = shared.clone();
thread::Builder::new()
.name("zoter manager".to_owned())
.spawn(move || serve(&socket, &shared))
};
let serving = match serving {
Ok(serving) => serving,
Err(e) => {
let _ = remove_file(&path);
return Err(e.into());
}
};
Ok(Self {
path,
shared,
serving: Some(serving),
activation: Vec::new(),
})
}
pub fn notify_socket(&self) -> &Path {
&self.path
}
pub fn set_store_max(&mut self, max: usize) {
self.shared.state().store_max = max;
}
pub fn store_max(&self) -> usize {
self.shared.state().store_max
}
pub fn add_socket(&mut self, name: &str, socket: OwnedFd) {
self.activation.push((name.to_owned(), socket));
}
pub fn take_sockets(&mut self) -> Vec<(String, OwnedFd)> {
std::mem::take(&mut self.activation)
}
pub fn passed(&self) -> Result<Passed> {
let state = self.shared.state();
let fds = self
.activation
.iter()
.chain(&state.store)
.map(|(name, fd)| Ok((name.clone(), fd.try_clone()?)))
.collect::<io::Result<_>>()?;
Ok(Passed::new(fds))
}
pub fn command(&self, program: impl AsRef<OsStr>) -> Result<Command> {
let passed = self.passed()?.into_vec();
let count = passed.len();
let first_free = RawFd::try_from(count)
.ok()
.and_then(|count| START.checked_add(count))
.ok_or_else(|| io::Error::from(ErrorKind::InvalidInput))?;
let mut names = Vec::with_capacity(count);
let mut copies = Vec::with_capacity(count);
for (name, fd) in passed {
names.push(name);
copies.push(fcntl_dupfd_cloexec(&fd, first_free).map_err(Error::os)?);
}
let sources: Vec<RawFd> = copies.iter().map(AsRawFd::as_raw_fd).collect();
let mut command = Command::new("sh");
command
.args(["-c", r#"LISTEN_PID=$$ exec "$0" "$@""#])
.arg(program);
for var in [
"LISTEN_PID",
"LISTEN_PIDFDID",
"LISTEN_FDS",
"LISTEN_FDNAMES",
] {
command.env_remove(var);
}
command.env("NOTIFY_SOCKET", &self.path);
match self.store_max() {
0 => command.env_remove("FDSTORE"),
max => command.env("FDSTORE", max.to_string()),
};
if count > 0 {
command
.env("LISTEN_FDS", count.to_string())
.env("LISTEN_FDNAMES", names.join(":"));
}
unsafe {
command.pre_exec(move || {
let _copies = &copies;
for (target, source) in (START..).zip(&sources) {
if libc::dup2(*source, target) < 0 {
return Err(io::Error::last_os_error());
}
}
Ok(())
});
}
Ok(command)
}
pub fn wait_for(&self, state: &str, timeout: Duration) -> Result<()> {
let deadline = Instant::now() + timeout;
let mut guard = self.shared.state();
loop {
while let Some(notification) = guard.notifications.pop_front() {
if notification.lines().any(|line| line == state) {
return Ok(());
}
}
let left = deadline.saturating_duration_since(Instant::now());
if left.is_zero() {
return Err(io::Error::new(
ErrorKind::TimedOut,
format!("no `{state}` came in time"),
)
.into());
}
guard = self
.shared
.changed
.wait_timeout(guard, left)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
}
pub fn barrier(&self, timeout: Duration) -> Result<()> {
Notifier::new(&self.path)?.barrier(timeout)
}
pub fn take_notifications(&self) -> Vec<String> {
self.shared.state().notifications.drain(..).collect()
}
pub fn stored_names(&self) -> Vec<String> {
let state = self.shared.state();
state.store.iter().map(|(name, _)| name.clone()).collect()
}
pub fn stored(&self) -> Result<Vec<(String, OwnedFd)>> {
let state = self.shared.state();
state
.store
.iter()
.map(|(name, fd)| Ok((name.clone(), fd.try_clone()?)))
.collect()
}
pub fn stored_fds(&self, key: &str) -> Result<usize> {
let name = format!("{key}/fds");
let mut state = self.shared.state();
let mut count = 0;
for (_, socket) in state.store.iter_mut().filter(|(stored, _)| *stored == name) {
let mut taken = Vec::new();
queue::take(socket.as_fd(), |index, fds, _| taken.push((index, fds)))?;
let refilled = Queue::new()?;
for (index, fds) in &taken {
let fds: Vec<_> = fds.iter().map(AsFd::as_fd).collect();
if !refilled.push(*index, &fds)? {
return Err(io::Error::other("a queue of the manager is full").into());
}
count += fds.len();
}
*socket = refilled.into_socket();
}
Ok(count)
}
pub fn replace_store(&self, fds: Vec<(String, OwnedFd)>) {
self.shared.state().store = fds;
}
pub fn take_removed(&self) -> Vec<String> {
std::mem::take(&mut self.shared.state().removed)
}
pub fn clear_store(&self) {
self.shared.state().store.clear();
}
}
impl fmt::Debug for Manager {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Manager")
.field("path", &self.path)
.field("activation", &self.activation)
.field("stored", &self.stored_names())
.finish_non_exhaustive()
}
}
impl Drop for Manager {
fn drop(&mut self) {
self.shared.state().stopping = true;
if let Ok(waking) = UnixDatagram::unbound() {
let _ = waking.send_to(&[], &self.path);
}
if let Some(serving) = self.serving.take() {
let _ = serving.join();
}
let _ = remove_file(&self.path);
}
}
pub fn store(notify_socket: impl AsRef<OsStr>, key: &str, max: usize) -> Result<Store> {
Store::with_notifier(Notifier::new(notify_socket)?, key, max)
}
struct Shared {
state: Mutex<State>,
changed: Condvar,
}
impl Shared {
fn state(&self) -> MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
}
#[derive(Default)]
struct State {
store: Vec<(String, OwnedFd)>,
store_max: usize,
removed: Vec<String>,
notifications: VecDeque<String>,
stopping: bool,
}
fn serve(socket: &UnixDatagram, shared: &Shared) {
loop {
let received = receive(socket);
let mut state = shared.state();
if state.stopping {
return;
}
let Ok((text, fds)) = received else {
continue;
};
process(&mut state, &text, fds);
state.notifications.push_back(text);
drop(state);
shared.changed.notify_all();
}
}
fn process(state: &mut State, text: &str, fds: Vec<OwnedFd>) {
let assignments: Vec<_> = text
.lines()
.filter_map(|line| line.split_once('='))
.collect();
let value = |key: &str| {
assignments
.iter()
.find(|(assigned, _)| *assigned == key)
.map(|(_, value)| *value)
};
let name = value("FDNAME");
if value("FDSTORE") == Some("1") {
let name = name.unwrap_or("stored");
for fd in fds {
let kept = state.store.iter().any(|(_, stored)| same_file(stored, &fd));
if !kept && state.store.len() < state.store_max {
state.store.push((name.to_owned(), fd));
}
}
} else if value("FDSTOREREMOVE") == Some("1")
&& let Some(name) = name
{
state.store.retain(|(stored, _)| stored != name);
state.removed.push(name.to_owned());
}
}
fn receive(socket: &UnixDatagram) -> io::Result<(String, Vec<OwnedFd>)> {
let mut text = [0; MAX_NOTIFICATION];
let mut space = [MaybeUninit::uninit(); cmsg_space!(ScmRights(MAX_FDS))];
let mut control = RecvAncillaryBuffer::new(&mut space);
let received = recvmsg(
socket,
&mut [IoSliceMut::new(&mut text)],
&mut control,
RecvFlags::CMSG_CLOEXEC,
)?;
let fds = control
.drain()
.filter_map(|message| match message {
RecvAncillaryMessage::ScmRights(fds) => Some(fds),
_ => None,
})
.flatten()
.collect();
let text = String::from_utf8_lossy(&text[..received.bytes]).into_owned();
Ok((text, fds))
}
fn same_file(a: &OwnedFd, b: &OwnedFd) -> bool {
let pid = libc::c_long::from(unsafe { libc::getpid() });
let order = unsafe {
libc::syscall(
libc::SYS_kcmp,
pid,
pid,
KCMP_FILE,
libc::c_long::from(a.as_fd().as_raw_fd()),
libc::c_long::from(b.as_fd().as_raw_fd()),
)
};
order == 0
}
fn unique_name() -> String {
static NEXT: AtomicUsize = AtomicUsize::new(0);
format!(
"zoter-manager-{}-{}",
process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
)
}
const DEFAULT_STORE_MAX: usize = 16;
const MAX_NOTIFICATION: usize = 4096;
const KCMP_FILE: libc::c_long = 0;
#[cfg(test)]
mod tests {
use std::{os::fd::AsFd, process::Stdio};
use rustix::io::write;
use super::*;
use crate::{
Error,
test_utils::{pipe, read_byte},
};
#[test]
fn store_and_remove() {
let manager = Manager::new().unwrap();
let notifier = Notifier::new(manager.notify_socket()).unwrap();
let (read, write_end) = pipe();
notifier.store("a", &[write_end.as_fd()]).unwrap();
notifier.store("b", &[write_end.as_fd()]).unwrap();
notifier.store("b", &[pipe().1.as_fd()]).unwrap();
notifier.barrier(WAIT).unwrap();
assert_eq!(manager.stored_names(), ["a", "b"]);
let (_, stored) = manager.stored().unwrap().remove(0);
write(&stored, b"x").unwrap();
assert_eq!(read_byte(&read), b'x');
notifier.remove("b").unwrap();
notifier.barrier(WAIT).unwrap();
assert_eq!(manager.stored_names(), ["a"]);
assert_eq!(manager.take_removed(), ["b"]);
assert!(manager.take_removed().is_empty());
manager.clear_store();
assert!(manager.stored_names().is_empty());
}
#[test]
fn store_takes_its_max() {
let mut manager = Manager::new().unwrap();
manager.set_store_max(2);
let notifier = Notifier::new(manager.notify_socket()).unwrap();
let fds: Vec<_> = (0..3).map(|_| pipe().1).collect();
let borrowed: Vec<_> = fds.iter().map(AsFd::as_fd).collect();
notifier.store("a", &borrowed).unwrap();
notifier.barrier(WAIT).unwrap();
assert_eq!(manager.stored_names(), ["a", "a"]);
}
#[test]
fn wait_for_notifications() {
let manager = Manager::new().unwrap();
let notifier = Notifier::new(manager.notify_socket()).unwrap();
notifier.notify("STATUS=Starting").unwrap();
notifier.notify("READY=1\nSTATUS=Up").unwrap();
manager.wait_for("READY=1", WAIT).unwrap();
let error = manager
.wait_for("READY=1", Duration::from_millis(100))
.unwrap_err();
assert!(
matches!(&error, Error::Io(e) if e.kind() == ErrorKind::TimedOut),
"{error:?}"
);
notifier.notify("STATUS=Down").unwrap();
manager.barrier(WAIT).unwrap();
assert_eq!(manager.take_notifications(), ["STATUS=Down", "BARRIER=1"]);
assert!(manager.take_notifications().is_empty());
}
#[test]
fn passed_as_systemd_passes_them() {
let mut manager = Manager::new().unwrap();
let notifier = Notifier::new(manager.notify_socket()).unwrap();
manager.add_socket("example.socket", pipe().1);
notifier.store("stored", &[pipe().1.as_fd()]).unwrap();
notifier.barrier(WAIT).unwrap();
let passed = manager.passed().unwrap();
let names: Vec<_> = passed.iter().map(|(name, _)| name).collect();
assert_eq!(names, ["example.socket", "stored"]);
assert_eq!(manager.take_sockets().len(), 1);
assert_eq!(manager.passed().unwrap().len(), 1);
}
#[test]
fn stored_fds() {
let manager = Manager::new().unwrap();
let store = super::store(manager.notify_socket(), "test", 2).unwrap();
let items = [(1, Fds(vec![pipe().1])), (2, Fds(vec![pipe().1, pipe().1]))];
store.hand_over(&[][..], items).unwrap();
assert_eq!(manager.stored_fds("test").unwrap(), 3);
assert_eq!(manager.stored_fds("test").unwrap(), 3);
assert_eq!(manager.stored_fds("other").unwrap(), 0);
let mut passed = manager.passed().unwrap();
let mut store = super::store(manager.notify_socket(), "test", 2).unwrap();
let mut restored = store.take_over(&mut passed).unwrap().unwrap();
let fds: Vec<_> = restored.take_items().map(|item| item.fds.len()).collect();
assert_eq!(fds, [1, 2]);
}
struct Fds(Vec<OwnedFd>);
impl crate::Save for Fds {
fn saved_len(&self) -> Option<usize> {
Some(0)
}
fn save(&self, _dest: &mut [u8]) -> Result<()> {
Ok(())
}
fn fds(&self) -> Vec<std::os::fd::BorrowedFd<'_>> {
self.0.iter().map(AsFd::as_fd).collect()
}
}
#[test]
fn command() {
let mut manager = Manager::new().unwrap();
manager.add_socket("first", pipe().1);
manager.add_socket("second", pipe().1);
let script = r#"
test "$LISTEN_PID" = "$$" && test -e /proc/$$/fd/3 && test -e /proc/$$/fd/4 &&
echo "$LISTEN_FDS $LISTEN_FDNAMES ${FDSTORE-unset} $NOTIFY_SOCKET $1"
"#;
for (max, told) in [(4, "4"), (0, "unset")] {
manager.set_store_max(max);
let output = manager
.command("sh")
.unwrap()
.args(["-c", script, "sh", "arg"])
.stdin(Stdio::null())
.output()
.unwrap();
assert!(output.status.success(), "{output:?}");
let expected = format!(
"2 first:second {told} {} arg\n",
manager.notify_socket().display()
);
assert_eq!(String::from_utf8_lossy(&output.stdout), expected);
}
}
const WAIT: Duration = Duration::from_secs(10);
}