use crate::ClockError;
use std::net::UdpSocket;
use std::time::Duration;
#[cfg(unix)]
pub fn wait_readable(socket: &UdpSocket, timeout: Duration) -> Result<bool, ClockError> {
use std::os::fd::AsRawFd;
let mut pfd = libc::pollfd {
fd: socket.as_raw_fd(),
events: libc::POLLIN,
revents: 0,
};
let timeout_ms = timeout.as_millis().min(i32::MAX as u128) as libc::c_int;
loop {
let rc = unsafe { libc::poll(&mut pfd, 1, timeout_ms) };
if rc > 0 {
return Ok(pfd.revents & (libc::POLLIN | libc::POLLERR | libc::POLLHUP) != 0);
}
if rc == 0 {
return Ok(false);
}
let err = std::io::Error::last_os_error();
if err.kind() == std::io::ErrorKind::Interrupted {
continue;
}
return Err(ClockError {
op: "poll",
detail: err.to_string(),
});
}
}
#[cfg(windows)]
pub fn wait_readable(_socket: &UdpSocket, _timeout: Duration) -> Result<bool, ClockError> {
Ok(true)
}
#[derive(Clone, Copy, Debug)]
pub struct Received {
pub len: usize,
pub peer: std::net::SocketAddr,
pub kernel_rx_ns: Option<i64>,
}
#[cfg(target_os = "linux")]
pub fn enable_rx_timestamps(socket: &UdpSocket) -> Result<(), ClockError> {
use std::os::fd::AsRawFd;
const RX_HARDWARE: libc::c_int = 1 << 0;
const RX_SOFTWARE: libc::c_int = 1 << 3;
const SOFTWARE: libc::c_int = 1 << 4;
const RAW_HARDWARE: libc::c_int = 1 << 6;
let flags: libc::c_int = RX_SOFTWARE | SOFTWARE | RX_HARDWARE | RAW_HARDWARE;
let rc = unsafe {
libc::setsockopt(
socket.as_raw_fd(),
libc::SOL_SOCKET,
libc::SO_TIMESTAMPING,
(&raw const flags).cast::<libc::c_void>(),
core::mem::size_of::<libc::c_int>() as libc::socklen_t,
)
};
if rc != 0 {
return Err(ClockError {
op: "setsockopt(SO_TIMESTAMPING)",
detail: std::io::Error::last_os_error().to_string(),
});
}
Ok(())
}
#[cfg(not(target_os = "linux"))]
pub fn enable_rx_timestamps(_socket: &UdpSocket) -> Result<(), ClockError> {
Err(ClockError {
op: "enable_rx_timestamps",
detail: "kernel receive timestamps are not implemented on this platform".into(),
})
}
pub fn activated_udp_socket() -> Option<UdpSocket> {
#[cfg(target_os = "linux")]
{
use std::os::fd::FromRawFd;
if !should_adopt_activated(
std::env::var("LISTEN_PID").ok().as_deref(),
std::env::var("LISTEN_FDS").ok().as_deref(),
std::process::id(),
) {
return None;
}
Some(unsafe { UdpSocket::from_raw_fd(3) })
}
#[cfg(not(target_os = "linux"))]
{
None
}
}
pub fn should_adopt_activated(
listen_pid: Option<&str>,
listen_fds: Option<&str>,
our_pid: u32,
) -> bool {
let for_us = listen_pid
.and_then(|p| p.parse::<u32>().ok())
.map(|pid| pid == our_pid)
.unwrap_or(false);
let count = listen_fds.and_then(|n| n.parse::<i32>().ok()).unwrap_or(0);
for_us && count >= 1
}
pub const BATCH_SIZE: usize = 32;
#[cfg(target_os = "linux")]
#[derive(Default)]
pub struct BatchScratch {
want_control: bool,
iovecs: Vec<libc::iovec>,
addrs: Vec<libc::sockaddr_storage>,
msgs: Vec<libc::mmsghdr>,
controls: Vec<[u8; CONTROL_LEN]>,
}
#[cfg(target_os = "linux")]
impl BatchScratch {
pub fn new() -> Self {
let mut scratch = BatchScratch {
want_control: true,
..BatchScratch::default()
};
scratch.ensure(BATCH_SIZE);
scratch
}
pub fn without_timestamps() -> Self {
let mut scratch = BatchScratch::default();
scratch.ensure(BATCH_SIZE);
scratch
}
fn ensure(&mut self, count: usize) {
if self.msgs.len() >= count {
return;
}
self.iovecs
.resize(count, unsafe { core::mem::zeroed::<libc::iovec>() });
self.addrs.resize(count, unsafe {
core::mem::zeroed::<libc::sockaddr_storage>()
});
self.msgs
.resize(count, unsafe { core::mem::zeroed::<libc::mmsghdr>() });
self.controls.resize(count, [0u8; CONTROL_LEN]);
}
}
#[cfg(not(target_os = "linux"))]
#[derive(Default)]
pub struct BatchScratch;
#[cfg(not(target_os = "linux"))]
impl BatchScratch {
pub fn new() -> Self {
BatchScratch
}
pub fn without_timestamps() -> Self {
BatchScratch
}
}
#[cfg(target_os = "linux")]
pub fn recv_batch(
socket: &UdpSocket,
bufs: &mut [[u8; 1024]],
scratch: &mut BatchScratch,
out: &mut Vec<Received>,
) -> Result<usize, ClockError> {
use std::os::fd::AsRawFd;
out.clear();
let count = bufs.len().min(BATCH_SIZE);
if count == 0 {
return Ok(0);
}
scratch.ensure(count);
let BatchScratch {
want_control,
iovecs,
addrs,
msgs,
controls,
} = scratch;
for i in 0..count {
iovecs[i] = libc::iovec {
iov_base: bufs[i].as_mut_ptr().cast::<libc::c_void>(),
iov_len: bufs[i].len(),
};
}
for i in 0..count {
let hdr = &mut msgs[i].msg_hdr;
hdr.msg_name = (&mut addrs[i] as *mut libc::sockaddr_storage).cast::<libc::c_void>();
hdr.msg_namelen = core::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t;
hdr.msg_iov = (&mut iovecs[i]) as *mut libc::iovec;
hdr.msg_iovlen = 1;
if *want_control {
hdr.msg_control = controls[i].as_mut_ptr().cast::<libc::c_void>();
hdr.msg_controllen = CONTROL_LEN as _;
} else {
hdr.msg_control = core::ptr::null_mut();
hdr.msg_controllen = 0;
}
msgs[i].msg_len = 0;
}
let received = unsafe {
libc::recvmmsg(
socket.as_raw_fd(),
msgs.as_mut_ptr(),
count as libc::c_uint,
libc::MSG_DONTWAIT as _,
core::ptr::null_mut(),
)
};
if received < 0 {
let err = std::io::Error::last_os_error();
if matches!(
err.kind(),
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
) {
return Ok(0);
}
return Err(ClockError {
op: "recvmmsg",
detail: err.to_string(),
});
}
for msg in msgs.iter().take(received as usize) {
let len = msg.msg_len as usize;
let peer =
unsafe { sockaddr_to_rust(&*(msg.msg_hdr.msg_name as *const libc::sockaddr_storage)) };
if let Some(peer) = peer {
let kernel_rx_ns = unsafe { kernel_timestamp(&msg.msg_hdr) };
out.push(Received {
len,
peer,
kernel_rx_ns,
});
}
}
Ok(out.len())
}
#[cfg(target_os = "linux")]
const CONTROL_LEN: usize = 128;
#[cfg(target_os = "linux")]
unsafe fn kernel_timestamp(hdr: &libc::msghdr) -> Option<i64> {
if hdr.msg_controllen == 0 || hdr.msg_control.is_null() {
return None;
}
let mut cmsg = unsafe { libc::CMSG_FIRSTHDR(hdr) };
while !cmsg.is_null() {
let header = unsafe { &*cmsg };
if header.cmsg_level == libc::SOL_SOCKET && header.cmsg_type == libc::SCM_TIMESTAMPING {
let stamps = unsafe { libc::CMSG_DATA(cmsg).cast::<libc::timespec>() };
for index in [0usize, 2] {
let ts = unsafe { *stamps.add(index) };
if ts.tv_sec != 0 || ts.tv_nsec != 0 {
return Some(ts.tv_sec as i64 * 1_000_000_000 + ts.tv_nsec as i64);
}
}
}
cmsg = unsafe { libc::CMSG_NXTHDR(hdr, cmsg) };
}
None
}
#[cfg(target_os = "linux")]
#[cfg(target_os = "linux")]
fn rust_to_sockaddr(
addr: &std::net::SocketAddr,
out: &mut libc::sockaddr_storage,
) -> libc::socklen_t {
match addr {
std::net::SocketAddr::V4(v4) => {
let sin = unsafe { &mut *(out as *mut _ as *mut libc::sockaddr_in) };
sin.sin_family = libc::AF_INET as libc::sa_family_t;
sin.sin_port = v4.port().to_be();
sin.sin_addr.s_addr = u32::from(*v4.ip()).to_be();
core::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t
}
std::net::SocketAddr::V6(v6) => {
let sin6 = unsafe { &mut *(out as *mut _ as *mut libc::sockaddr_in6) };
sin6.sin6_family = libc::AF_INET6 as libc::sa_family_t;
sin6.sin6_port = v6.port().to_be();
sin6.sin6_addr.s6_addr = v6.ip().octets();
sin6.sin6_flowinfo = v6.flowinfo();
sin6.sin6_scope_id = v6.scope_id();
core::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t
}
}
}
#[cfg(target_os = "linux")]
pub fn send_batch(
socket: &UdpSocket,
messages: &[(&[u8], std::net::SocketAddr)],
scratch: &mut BatchScratch,
) -> Result<usize, ClockError> {
send_batch_by(socket, messages, |m| m.0, |m| m.1, scratch)
}
#[cfg(target_os = "linux")]
pub fn send_batch_by<T>(
socket: &UdpSocket,
items: &[T],
payload: impl Fn(&T) -> &[u8],
peer: impl Fn(&T) -> std::net::SocketAddr,
scratch: &mut BatchScratch,
) -> Result<usize, ClockError> {
use std::os::fd::AsRawFd;
let count = items.len().min(BATCH_SIZE);
if count == 0 {
return Ok(0);
}
scratch.ensure(count);
let BatchScratch {
iovecs,
addrs,
msgs,
..
} = scratch;
for (i, item) in items.iter().enumerate().take(count) {
let bytes = payload(item);
iovecs[i] = libc::iovec {
iov_base: bytes.as_ptr() as *mut libc::c_void,
iov_len: bytes.len(),
};
let namelen = rust_to_sockaddr(&peer(item), &mut addrs[i]);
let hdr = &mut msgs[i].msg_hdr;
hdr.msg_name = (&mut addrs[i] as *mut libc::sockaddr_storage).cast::<libc::c_void>();
hdr.msg_namelen = namelen;
hdr.msg_iov = (&mut iovecs[i]) as *mut libc::iovec;
hdr.msg_iovlen = 1;
hdr.msg_control = core::ptr::null_mut();
hdr.msg_controllen = 0;
msgs[i].msg_len = 0;
}
let sent = unsafe {
libc::sendmmsg(
socket.as_raw_fd(),
msgs.as_mut_ptr(),
count as libc::c_uint,
libc::MSG_DONTWAIT as _,
)
};
if sent < 0 {
let err = std::io::Error::last_os_error();
if matches!(
err.kind(),
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
) {
return Ok(0);
}
return Err(ClockError {
op: "sendmmsg",
detail: err.to_string(),
});
}
Ok(sent as usize)
}
#[cfg(not(target_os = "linux"))]
pub fn send_batch(
socket: &UdpSocket,
messages: &[(&[u8], std::net::SocketAddr)],
scratch: &mut BatchScratch,
) -> Result<usize, ClockError> {
send_batch_by(socket, messages, |m| m.0, |m| m.1, scratch)
}
#[cfg(not(target_os = "linux"))]
pub fn send_batch_by<T>(
socket: &UdpSocket,
items: &[T],
payload: impl Fn(&T) -> &[u8],
peer: impl Fn(&T) -> std::net::SocketAddr,
_scratch: &mut BatchScratch,
) -> Result<usize, ClockError> {
let mut sent = 0;
for item in items {
match socket.send_to(payload(item), peer(item)) {
Ok(_) => sent += 1,
Err(e)
if matches!(
e.kind(),
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
) =>
{
break;
}
Err(e) => {
return Err(ClockError {
op: "send_to",
detail: e.to_string(),
});
}
}
}
Ok(sent)
}
#[cfg(target_os = "linux")]
fn sockaddr_to_rust(storage: &libc::sockaddr_storage) -> Option<std::net::SocketAddr> {
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6};
match storage.ss_family as libc::c_int {
libc::AF_INET => {
let sin = unsafe { &*(storage as *const _ as *const libc::sockaddr_in) };
let ip = Ipv4Addr::from(u32::from_be(sin.sin_addr.s_addr));
Some(std::net::SocketAddr::V4(SocketAddrV4::new(
ip,
u16::from_be(sin.sin_port),
)))
}
libc::AF_INET6 => {
let sin6 = unsafe { &*(storage as *const _ as *const libc::sockaddr_in6) };
let ip = Ipv6Addr::from(sin6.sin6_addr.s6_addr);
Some(std::net::SocketAddr::V6(SocketAddrV6::new(
ip,
u16::from_be(sin6.sin6_port),
sin6.sin6_flowinfo,
sin6.sin6_scope_id,
)))
}
_ => None,
}
}
#[cfg(not(target_os = "linux"))]
pub fn recv_batch(
socket: &UdpSocket,
bufs: &mut [[u8; 1024]],
_scratch: &mut BatchScratch,
out: &mut Vec<Received>,
) -> Result<usize, ClockError> {
out.clear();
let Some(buf) = bufs.first_mut() else {
return Ok(0);
};
match socket.recv_from(buf) {
Ok((len, peer)) => {
out.push(Received {
len,
peer,
kernel_rx_ns: None,
});
Ok(1)
}
Err(e)
if matches!(
e.kind(),
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
) =>
{
Ok(0)
}
Err(e) => Err(ClockError {
op: "recv_from",
detail: e.to_string(),
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn timeout_and_readable_paths() {
let a = UdpSocket::bind("127.0.0.1:0").expect("bind a");
let b = UdpSocket::bind("127.0.0.1:0").expect("bind b");
let quick = wait_readable(&a, Duration::from_millis(50)).expect("wait");
if cfg!(unix) {
assert!(!quick, "empty socket reported readable");
} else {
assert!(quick);
}
b.send_to(b"x", a.local_addr().expect("addr"))
.expect("send");
let ready = wait_readable(&a, Duration::from_millis(2000)).expect("wait");
assert!(ready, "pending datagram not reported readable");
}
#[test]
fn batch_receive_returns_every_datagram_with_its_sender() {
let server = UdpSocket::bind("127.0.0.1:0").expect("bind server");
let server_addr = server.local_addr().expect("addr");
server
.set_read_timeout(Some(Duration::from_millis(500)))
.expect("timeout");
let client = UdpSocket::bind("127.0.0.1:0").expect("bind client");
let client_addr = client.local_addr().expect("client addr");
let sent = 5usize;
for i in 0..sent {
client.send_to(&[i as u8; 16], server_addr).expect("send");
}
let mut bufs = [[0u8; 1024]; BATCH_SIZE];
let mut out = Vec::new();
let mut total = 0usize;
let mut idle = 0;
while total < sent && idle < 20 {
if !wait_readable(&server, Duration::from_millis(200)).expect("wait") {
idle += 1;
continue;
}
let n =
recv_batch(&server, &mut bufs, &mut BatchScratch::new(), &mut out).expect("batch");
if n == 0 {
idle += 1;
continue;
}
for received in out.iter().take(n) {
assert_eq!(received.len, 16, "wrong length reported");
assert_eq!(
received.peer, client_addr,
"batch reported the wrong sender"
);
}
total += n;
}
assert_eq!(total, sent, "batch receive lost datagrams");
}
#[test]
fn activation_is_refused_unless_the_descriptors_are_ours() {
assert!(should_adopt_activated(Some("42"), Some("1"), 42));
assert!(!should_adopt_activated(Some("41"), Some("1"), 42));
assert!(!should_adopt_activated(None, None, 42));
assert!(!should_adopt_activated(Some("42"), Some("0"), 42));
assert!(!should_adopt_activated(Some("not-a-pid"), Some("1"), 42));
assert!(!should_adopt_activated(Some("42"), Some("many"), 42));
}
#[test]
fn batch_receive_on_an_empty_socket_returns_zero_not_an_error() {
let s = UdpSocket::bind("127.0.0.1:0").expect("bind");
s.set_read_timeout(Some(Duration::from_millis(50)))
.expect("timeout");
let mut bufs = [[0u8; 1024]; 4];
let mut out = Vec::new();
assert_eq!(
recv_batch(&s, &mut bufs, &mut BatchScratch::new(), &mut out).expect("batch"),
0
);
assert!(out.is_empty());
}
}