use std::collections::VecDeque;
use std::ffi::c_int;
use std::io;
use std::net::SocketAddr;
use std::sync::Mutex;
use crate::readiness::Deadline;
use crate::scope;
use crate::sockets::{SockRec, SocketKind};
pub(crate) const CAP_NET_BIND_SERVICE: c_int = 10;
pub(crate) const CAP_NET_ADMIN: c_int = 12;
pub(crate) const CAP_NET_RAW: c_int = 13;
pub(crate) const CAP_IPC_LOCK: c_int = 14;
#[cfg(unix)]
pub(crate) const CAP_SYS_ADMIN: c_int = 21;
pub(crate) const CAP_SYS_NICE: c_int = 23;
pub(crate) const CAP_SYS_RESOURCE: c_int = 24;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Rlimit {
pub cur: u64,
pub max: u64,
}
impl Rlimit {
pub const INFINITY: u64 = u64::MAX;
pub const fn new(limit: u64) -> Self {
Rlimit {
cur: limit,
max: limit,
}
}
pub const fn unlimited() -> Self {
Self::new(Self::INFINITY)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Privileges {
pub root: bool,
pub net_admin: bool,
pub net_raw: bool,
pub net_bind_service: bool,
pub sys_nice: bool,
pub ipc_lock: bool,
pub sys_resource: bool,
pub rtprio_limit: Rlimit,
pub nice_limit: Rlimit,
pub memlock_limit: Rlimit,
}
const STOCK_MEMLOCK: u64 = if cfg!(target_os = "linux") {
8 * 1024 * 1024
} else {
Rlimit::INFINITY
};
impl Privileges {
pub const fn all() -> Self {
Privileges {
root: true,
net_admin: true,
net_raw: true,
net_bind_service: true,
sys_nice: true,
ipc_lock: true,
sys_resource: true,
rtprio_limit: Rlimit::new(99),
nice_limit: Rlimit::new(40),
memlock_limit: Rlimit::unlimited(),
}
}
pub const fn none() -> Self {
Privileges {
root: false,
net_admin: false,
net_raw: false,
net_bind_service: false,
sys_nice: false,
ipc_lock: false,
sys_resource: false,
rtprio_limit: Rlimit::new(0),
nice_limit: Rlimit::new(0),
memlock_limit: Rlimit::new(STOCK_MEMLOCK),
}
}
#[cfg(unix)]
pub fn from_real_process() -> io::Result<Self> {
snare_interpose::real(|| {
let root = unsafe { libc::geteuid() } == 0;
let mut p = if cfg!(target_os = "linux") {
let status = std::fs::read_to_string("/proc/self/status")?;
let caps = status
.lines()
.find_map(|l| l.strip_prefix("CapEff:"))
.and_then(|v| u64::from_str_radix(v.trim(), 16).ok())
.ok_or_else(|| io::Error::from(io::ErrorKind::InvalidData))?;
Self::from_caps(caps, root)
} else if root {
Self::all()
} else {
Self::none()
};
#[cfg(target_os = "linux")]
{
p.rtprio_limit = crate::proclimits::real_rlimit(libc::RLIMIT_RTPRIO as c_int)?;
p.nice_limit = crate::proclimits::real_rlimit(libc::RLIMIT_NICE as c_int)?;
}
p.memlock_limit = crate::proclimits::real_rlimit(libc::RLIMIT_MEMLOCK as c_int)?;
Ok(p)
})
}
pub fn has_cap(&self, cap: c_int) -> bool {
match cap {
CAP_NET_BIND_SERVICE => self.net_bind_service,
CAP_NET_ADMIN => self.net_admin,
CAP_NET_RAW => self.net_raw,
CAP_IPC_LOCK => self.ipc_lock,
CAP_SYS_NICE => self.sys_nice,
CAP_SYS_RESOURCE => self.sys_resource,
_ => false,
}
}
#[cfg_attr(not(unix), allow(dead_code))]
pub(crate) fn from_caps(caps: u64, root: bool) -> Self {
let has = |cap: c_int| caps & (1u64 << cap) != 0;
Privileges {
root,
net_admin: has(CAP_NET_ADMIN),
net_raw: has(CAP_NET_RAW),
net_bind_service: has(CAP_NET_BIND_SERVICE),
sys_nice: has(CAP_SYS_NICE),
ipc_lock: has(CAP_IPC_LOCK),
sys_resource: has(CAP_SYS_RESOURCE),
..Self::none()
}
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub(crate) fn cap_mask(&self) -> u64 {
[
CAP_NET_BIND_SERVICE,
CAP_NET_ADMIN,
CAP_NET_RAW,
CAP_IPC_LOCK,
CAP_SYS_NICE,
CAP_SYS_RESOURCE,
]
.into_iter()
.filter(|&cap| self.has_cap(cap))
.fold(0, |mask, cap| mask | 1u64 << cap)
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SysLimits {
pub rmem_default: usize,
pub rmem_max: usize,
pub wmem_default: usize,
pub wmem_max: usize,
pub tcp_rmem_default: usize,
pub tcp_wmem_default: usize,
pub sockbuf_reject_at: usize,
pub unprivileged_port_start: u16,
pub enforce_rcvbuf: bool,
pub tcp_syn_retries: u8,
pub tcp_syn_linear_timeouts: u8,
pub listen_backlog_max: usize,
pub bindv6only: bool,
pub tcp_gso_max_size: usize,
pub udp_rcvbuf_overshoot: bool,
pub skb_small_truesize: usize,
pub skb_head_overhead: usize,
}
impl Default for SysLimits {
fn default() -> Self {
Self::host()
}
}
impl SysLimits {
pub fn host() -> Self {
if cfg!(target_os = "macos") {
SysLimits {
rmem_default: 786_896,
rmem_max: 8_388_608,
wmem_default: 9216,
wmem_max: 8_388_608,
tcp_rmem_default: 131_072,
tcp_wmem_default: 131_072,
sockbuf_reject_at: usize::MAX,
unprivileged_port_start: 1024,
enforce_rcvbuf: true,
tcp_syn_retries: 6,
tcp_syn_linear_timeouts: 0,
listen_backlog_max: 128,
bindv6only: false,
tcp_gso_max_size: 0,
udp_rcvbuf_overshoot: false,
skb_small_truesize: 960,
skb_head_overhead: 378,
}
} else if cfg!(target_os = "linux") {
SysLimits {
rmem_default: 212_992,
rmem_max: 4 << 20,
wmem_default: 212_992,
wmem_max: 4 << 20,
tcp_rmem_default: 131_072,
tcp_wmem_default: 16_384,
sockbuf_reject_at: usize::MAX,
unprivileged_port_start: 1024,
enforce_rcvbuf: true,
tcp_syn_retries: 6,
tcp_syn_linear_timeouts: 4,
listen_backlog_max: 4096,
bindv6only: false,
tcp_gso_max_size: 65536,
udp_rcvbuf_overshoot: false,
skb_small_truesize: 960,
skb_head_overhead: 378,
}
} else {
SysLimits {
rmem_default: 65_536,
rmem_max: usize::MAX,
wmem_default: 65_536,
wmem_max: usize::MAX,
tcp_rmem_default: 65_536,
tcp_wmem_default: 65_536,
sockbuf_reject_at: usize::MAX,
unprivileged_port_start: 0,
enforce_rcvbuf: true,
tcp_syn_retries: 6,
tcp_syn_linear_timeouts: 0,
listen_backlog_max: 200,
bindv6only: true,
tcp_gso_max_size: 0,
udp_rcvbuf_overshoot: false,
skb_small_truesize: 960,
skb_head_overhead: 378,
}
}
}
pub fn from_real_host() -> io::Result<Self> {
snare_interpose::real(Self::read_real)
}
#[cfg(target_os = "linux")]
fn read_real() -> io::Result<Self> {
let read = |path: &str| -> io::Result<Vec<usize>> {
let text = std::fs::read_to_string(path)?;
text.split_whitespace()
.map(|v| {
v.parse()
.map_err(|_| io::Error::from(io::ErrorKind::InvalidData))
})
.collect()
};
let one = |path: &str| -> io::Result<usize> {
read(path)?
.first()
.copied()
.ok_or_else(|| io::Error::from(io::ErrorKind::InvalidData))
};
let middle = |path: &str| -> io::Result<usize> {
read(path)?
.get(1)
.copied()
.ok_or_else(|| io::Error::from(io::ErrorKind::InvalidData))
};
let stock = Self::host();
let (skb_small_truesize, skb_head_overhead) =
probe_truesize().unwrap_or((stock.skb_small_truesize, stock.skb_head_overhead));
Ok(SysLimits {
rmem_default: one("/proc/sys/net/core/rmem_default")?,
rmem_max: one("/proc/sys/net/core/rmem_max")?,
wmem_default: one("/proc/sys/net/core/wmem_default")?,
wmem_max: one("/proc/sys/net/core/wmem_max")?,
tcp_rmem_default: middle("/proc/sys/net/ipv4/tcp_rmem")?,
tcp_wmem_default: middle("/proc/sys/net/ipv4/tcp_wmem")?,
sockbuf_reject_at: usize::MAX,
unprivileged_port_start: one("/proc/sys/net/ipv4/ip_unprivileged_port_start")?
.min(u16::MAX as usize) as u16,
enforce_rcvbuf: true,
tcp_syn_retries: one("/proc/sys/net/ipv4/tcp_syn_retries")?.min(u8::MAX as usize) as u8,
tcp_syn_linear_timeouts: match one("/proc/sys/net/ipv4/tcp_syn_linear_timeouts") {
Ok(value) => value.min(u8::MAX as usize) as u8,
Err(error) if error.kind() == io::ErrorKind::NotFound => 0,
Err(error) => return Err(error),
},
listen_backlog_max: one("/proc/sys/net/core/somaxconn")?,
bindv6only: match one("/proc/sys/net/ipv6/bindv6only") {
Ok(value) => value != 0,
Err(error) if error.kind() == io::ErrorKind::NotFound => false,
Err(error) => return Err(error),
},
tcp_gso_max_size: stock.tcp_gso_max_size,
udp_rcvbuf_overshoot: kernel_before(
&std::fs::read_to_string("/proc/sys/kernel/osrelease")?,
(6, 18),
),
skb_small_truesize,
skb_head_overhead,
})
}
#[cfg(target_os = "macos")]
fn read_real() -> io::Result<Self> {
fn sysctl(name: &std::ffi::CStr) -> io::Result<usize> {
let mut value = [0u8; 8];
let mut len = value.len();
let rc = unsafe {
libc::sysctlbyname(
name.as_ptr(),
value.as_mut_ptr().cast(),
&mut len,
std::ptr::null_mut(),
0,
)
};
if rc != 0 {
return Err(io::Error::last_os_error());
}
Ok(match len {
4 => u32::from_ne_bytes(value[..4].try_into().unwrap()) as usize,
_ => u64::from_ne_bytes(value) as usize,
})
}
let max = sysctl(c"kern.ipc.maxsockbuf")?;
Ok(SysLimits {
rmem_default: sysctl(c"net.inet.udp.recvspace")?,
rmem_max: max,
wmem_default: sysctl(c"net.inet.udp.maxdgram")?,
wmem_max: max,
tcp_rmem_default: sysctl(c"net.inet.tcp.recvspace")?,
tcp_wmem_default: sysctl(c"net.inet.tcp.sendspace")?,
listen_backlog_max: sysctl(c"kern.ipc.somaxconn")?,
bindv6only: sysctl(c"net.inet6.ip6.v6only")? != 0,
..Self::host()
})
}
#[cfg(windows)]
fn read_real() -> io::Result<Self> {
use std::os::windows::io::AsRawSocket;
use windows_sys::Win32::Networking::WinSock::{
AF_INET, INVALID_SOCKET, IPPROTO_TCP, SO_RCVBUF, SO_SNDBUF, SOCK_STREAM, SOCKET,
SOL_SOCKET, closesocket, getsockopt, socket,
};
fn buffer(socket: SOCKET, name: i32) -> io::Result<usize> {
let mut value = 0i32;
let mut len = size_of::<i32>() as i32;
let rc = unsafe {
getsockopt(
socket,
SOL_SOCKET,
name,
(&mut value as *mut i32).cast(),
&mut len,
)
};
if rc != 0 {
return Err(io::Error::last_os_error());
}
usize::try_from(value).map_err(|_| io::Error::from(io::ErrorKind::InvalidData))
}
let udp_socket = std::net::UdpSocket::bind((std::net::Ipv4Addr::LOCALHOST, 0))?;
let udp = udp_socket.as_raw_socket() as SOCKET;
let tcp = unsafe { socket(AF_INET as i32, SOCK_STREAM, IPPROTO_TCP) };
if tcp == INVALID_SOCKET {
return Err(io::Error::last_os_error());
}
let tcp_buffers = buffer(tcp, SO_RCVBUF).and_then(|r| Ok((r, buffer(tcp, SO_SNDBUF)?)));
unsafe { closesocket(tcp) };
let (tcp_rmem_default, tcp_wmem_default) = tcp_buffers?;
Ok(SysLimits {
rmem_default: buffer(udp, SO_RCVBUF)?,
wmem_default: buffer(udp, SO_SNDBUF)?,
tcp_rmem_default,
tcp_wmem_default,
..Self::host()
})
}
}
#[cfg(unix)]
pub(crate) const UNPRIVILEGED_UID: u32 = if cfg!(target_os = "macos") { 501 } else { 1000 };
#[cfg(unix)]
pub(crate) const UNPRIVILEGED_GID: u32 = if cfg!(target_os = "macos") { 20 } else { 1000 };
#[cfg(unix)]
const DEFAULT_UMASK: u32 = 0o022;
pub(crate) struct SysConfig {
privileges: Mutex<Privileges>,
limits: Mutex<SysLimits>,
#[cfg(unix)]
wired: Mutex<crate::proclimits::Wired>,
#[cfg(unix)]
umask: std::sync::atomic::AtomicU32,
#[cfg(unix)]
identity: std::sync::atomic::AtomicBool,
}
impl SysConfig {
pub(crate) fn new() -> Self {
SysConfig {
privileges: Mutex::new(Privileges::all()),
limits: Mutex::new(SysLimits::host()),
#[cfg(unix)]
wired: Mutex::new(crate::proclimits::Wired::default()),
#[cfg(unix)]
umask: std::sync::atomic::AtomicU32::new(DEFAULT_UMASK),
#[cfg(unix)]
identity: std::sync::atomic::AtomicBool::new(false),
}
}
#[cfg(unix)]
pub(crate) fn umask(&self, mask: Option<u32>) -> u32 {
use std::sync::atomic::Ordering;
match mask {
Some(mask) => self.umask.swap(mask & 0o777, Ordering::AcqRel),
None => self.umask.load(Ordering::Acquire),
}
}
#[cfg(unix)]
pub(crate) fn model_identity(&self) {
self.identity
.store(true, std::sync::atomic::Ordering::Release);
}
#[cfg(unix)]
pub(crate) fn identity(&self) -> Option<(u32, u32)> {
if !self.identity.load(std::sync::atomic::Ordering::Acquire) {
return None;
}
Some(if self.privileges().root {
(0, 0)
} else {
(UNPRIVILEGED_UID, UNPRIVILEGED_GID)
})
}
#[cfg(unix)]
pub(crate) fn wired(&self) -> std::sync::MutexGuard<'_, crate::proclimits::Wired> {
snare_interpose::real(|| self.wired.lock().unwrap_or_else(|e| e.into_inner()))
}
pub(crate) fn privileges(&self) -> Privileges {
snare_interpose::real(|| self.privileges.lock().unwrap().clone())
}
pub(crate) fn set_privileges(&self, change: impl FnOnce(&mut Privileges)) {
snare_interpose::real(|| change(&mut self.privileges.lock().unwrap()));
}
#[cfg_attr(windows, allow(dead_code))]
pub(crate) fn has_cap(&self, cap: c_int) -> bool {
self.privileges().has_cap(cap)
}
pub(crate) fn limits(&self) -> SysLimits {
snare_interpose::real(|| self.limits.lock().unwrap().clone())
}
pub(crate) fn set_limits(&self, change: impl FnOnce(&mut SysLimits)) {
snare_interpose::real(|| change(&mut self.limits.lock().unwrap()));
}
#[cfg_attr(windows, allow(dead_code))]
pub(crate) fn bind_denied(&self, addr: SocketAddr) -> Option<c_int> {
let port = addr.port();
if port == 0 || port >= self.limits().unprivileged_port_start {
return None;
}
let privileges = self.privileges();
let allowed = if cfg!(target_os = "linux") {
privileges.net_bind_service
} else if cfg!(target_os = "macos") {
privileges.root || addr.ip().is_unspecified()
} else {
true
};
(!allowed).then_some(crate::netif::code::EACCES)
}
}
#[track_caller]
pub fn privileges() -> Privileges {
scope::here().sys.privileges()
}
#[track_caller]
pub fn set_privileges(change: impl FnOnce(&mut Privileges)) {
scope::here().sys.set_privileges(change);
}
#[track_caller]
pub fn sys_limits() -> SysLimits {
scope::here().sys.limits()
}
#[track_caller]
pub fn set_sys_limits(change: impl FnOnce(&mut SysLimits)) {
scope::here().sys.set_limits(change);
}
#[derive(Clone, Copy, Default)]
pub(crate) struct Charge {
mem: usize,
cc: usize,
}
#[derive(Clone, Copy, Default)]
pub(crate) struct RxAccount {
pub(crate) rmem_alloc: usize,
pub(crate) queued: usize,
pub(crate) queued_bytes: usize,
pub(crate) cc: usize,
seen: bool,
}
impl RxAccount {
fn charge(&self, len: usize, v6: bool, skb: (usize, usize), mtu: Option<u32>) -> Charge {
if cfg!(target_os = "linux") {
Charge {
mem: linux_truesize_fragmented(len, v6, skb, mtu),
cc: 0,
}
} else if cfg!(target_os = "macos") {
Charge {
mem: mac::mbufs(len),
cc: mac::record(len, self.seen),
}
} else {
Charge { mem: len, cc: 0 }
}
}
fn fits(&self, rcvbuf: i32, sb_max: usize, overshoot: bool, charge: Charge) -> bool {
let rcvbuf = rcvbuf.max(0) as usize;
if cfg!(target_os = "linux") {
if overshoot {
self.rmem_alloc <= rcvbuf
} else {
self.rmem_alloc == 0 || self.rmem_alloc + charge.mem <= rcvbuf
}
} else if cfg!(target_os = "macos") {
let mbmax = rcvbuf.saturating_mul(mac::SB_EFFICIENCY).min(sb_max);
charge.cc <= rcvbuf.saturating_sub(self.cc)
&& self.rmem_alloc < mbmax.saturating_add(mac::MBUF_SLACK)
} else {
self.queued_bytes < rcvbuf
}
}
fn take(&mut self, len: usize, charge: Charge) {
self.rmem_alloc += charge.mem;
self.cc += charge.cc;
self.queued += 1;
self.queued_bytes += len;
self.seen = true;
}
fn give_back(&mut self, len: usize, charge: Charge) {
self.rmem_alloc = self.rmem_alloc.saturating_sub(charge.mem);
self.cc = self.cc.saturating_sub(charge.cc);
self.queued = self.queued.saturating_sub(1);
self.queued_bytes = self.queued_bytes.saturating_sub(len);
}
}
#[cfg(target_os = "linux")]
fn kernel_before(release: &str, (major, minor): (u32, u32)) -> bool {
let mut parts = release
.trim()
.split(|c: char| !c.is_ascii_digit())
.map(str::parse::<u32>);
match (parts.next(), parts.next()) {
(Some(Ok(a)), Some(Ok(b))) => (a, b) < (major, minor),
_ => false,
}
}
const SK_BUFF_SIZE: usize = 256;
pub(crate) fn linux_truesize(len: usize, v6: bool, (small, overhead): (usize, usize)) -> usize {
let need = len + if v6 { 13 } else { 0 } + overhead;
if need < small - SK_BUFF_SIZE {
return small;
}
match [1024, 2048, 4096, 8192]
.into_iter()
.find(|&class| need < class)
{
Some(class) => class + SK_BUFF_SIZE,
None if need < 16384 - 1 => 16384 + SK_BUFF_SIZE,
None => len + small,
}
}
pub(crate) fn linux_truesize_fragmented(
len: usize,
v6: bool,
skb: (usize, usize),
mtu: Option<u32>,
) -> usize {
const UDP_HEADER: usize = 8;
let ip_header = if v6 { 40 + 8 } else { 20 };
let unfragmented = if v6 { 40 } else { 20 } + UDP_HEADER + len;
let Some(mtu) = mtu
.map(|m| m as usize)
.filter(|&m| unfragmented > m && m > ip_header + 8)
else {
return linux_truesize(len, v6, skb);
};
let max_fragment = (mtu - ip_header) & !7;
let mut rest = UDP_HEADER + len;
let mut total = 0;
while rest > 0 {
let fragment = rest.min(max_fragment);
rest -= fragment;
let as_datagram = if v6 {
fragment
} else {
fragment.saturating_sub(UDP_HEADER)
};
total += linux_truesize(as_datagram, v6, skb);
}
total
}
#[cfg(target_os = "linux")]
fn probe_truesize() -> Option<(usize, usize)> {
use std::os::fd::AsRawFd;
const SO_MEMINFO: c_int = 55;
let rx = std::net::UdpSocket::bind("127.0.0.1:0").ok()?;
let tx = std::net::UdpSocket::bind("127.0.0.1:0").ok()?;
rx.set_read_timeout(Some(std::time::Duration::from_secs(1)))
.ok()?;
let to = rx.local_addr().ok()?;
let buf = [0u8; 1024];
let truesize = |len: usize| -> Option<usize> {
tx.send_to(&buf[..len], to).ok()?;
let mut peek = [0u8; 1];
rx.peek(&mut peek).ok()?;
let mut words = [0u32; 9];
let mut optlen = std::mem::size_of_val(&words) as libc::socklen_t;
let rc = unsafe {
libc::getsockopt(
rx.as_raw_fd(),
libc::SOL_SOCKET,
SO_MEMINFO,
words.as_mut_ptr().cast(),
&mut optlen,
)
};
let mut sink = [0u8; 2048];
rx.recv(&mut sink).ok()?;
(rc == 0).then_some(words[0] as usize)
};
let small = truesize(0)?;
let (mut lo, mut hi) = (0, 1024);
while lo < hi {
let mid = (lo + hi) / 2;
if truesize(mid)? >= 2048 + SK_BUFF_SIZE {
hi = mid;
} else {
lo = mid + 1;
}
}
(lo > 0 && lo < 1024).then_some((small, 1024 - lo))
}
mod mac {
pub(super) const SB_EFFICIENCY: usize = 8;
pub(super) const MBUF_SLACK: usize = 448;
pub(super) fn record(len: usize, seen: bool) -> usize {
len + 16 + if seen { 16 } else { 0 }
}
pub(super) fn mbufs(len: usize) -> usize {
if len <= 200 { 1536 } else { 3584 }
}
}
pub(crate) trait Arrival {
fn src(&self) -> SocketAddr;
fn payload(&self) -> usize;
fn arrives(&self) -> Option<Deadline>;
fn lost(&self) -> bool;
fn path_mtu(&self) -> Option<u32> {
None
}
}
struct Queued<D> {
dg: D,
charge: Charge,
drops: u32,
}
pub(crate) struct RxQueue<D> {
in_flight: Vec<(u64, D)>,
rx: VecDeque<Queued<D>>,
seq: u64,
landed: u64,
}
impl<D> Default for RxQueue<D> {
fn default() -> Self {
RxQueue {
in_flight: Vec::new(),
rx: VecDeque::new(),
seq: 0,
landed: 0,
}
}
}
impl<D: Arrival> RxQueue<D> {
pub(crate) fn pending_time(&self) -> bool {
!self.in_flight.is_empty()
}
pub(crate) fn push(&mut self, dg: D, rec: Option<&SockRec>) {
self.seq += 1;
if let Some(arrives) = dg.arrives() {
self.in_flight.push((self.seq, dg));
#[cfg(target_os = "linux")]
if let Some(rec) = rec {
rec.note_data_arrival(arrives);
}
#[cfg(not(target_os = "linux"))]
let _ = arrives;
return;
}
self.land(rec);
self.enqueue(dg, rec);
}
pub(crate) fn land(&mut self, rec: Option<&SockRec>) {
self.land_when(rec, |dg| dg.arrives().is_none_or(|d| d.passed()));
}
#[cfg(windows)]
fn land_at(&mut self, rec: Option<&SockRec>, cutoff: std::time::Duration) {
self.land_when(rec, |dg| dg.arrives().is_none_or(|d| d.at() <= cutoff));
}
fn land_when(&mut self, rec: Option<&SockRec>, arrived: impl Fn(&D) -> bool) {
if self.in_flight.is_empty() {
return;
}
let (mut due, rest): (Vec<_>, Vec<_>) = std::mem::take(&mut self.in_flight)
.into_iter()
.partition(|(_, dg)| arrived(dg));
self.in_flight = rest;
due.retain(|(_, dg)| !dg.lost());
due.sort_by_key(|(seq, dg)| (dg.arrives().map(|d| d.instant()), *seq));
for (_, dg) in due {
self.enqueue(dg, rec);
}
#[cfg(target_os = "linux")]
self.track_arrival(rec);
}
#[cfg(target_os = "linux")]
fn track_arrival(&self, rec: Option<&SockRec>) {
if let Some(rec) = rec {
let arrives = self
.in_flight
.iter()
.filter_map(|(_, d)| d.arrives())
.min_by_key(|d| d.at());
rec.set_data_arrival(arrives);
}
}
fn enqueue(&mut self, dg: D, rec: Option<&SockRec>) {
let admitted = match rec {
Some(rec) => rec.admit(dg.payload(), dg.src(), dg.path_mtu(), dg.arrives()),
None => Some((Charge::default(), 0)),
};
if let Some((charge, drops)) = admitted {
self.rx.push_back(Queued { dg, charge, drops });
self.landed += 1;
}
}
pub(crate) fn pop(
&mut self,
accept: impl Fn(SocketAddr) -> bool,
rec: Option<&SockRec>,
) -> Option<(D, u32)> {
self.land(rec);
self.pop_landed(accept, rec)
}
pub(crate) fn pop_landed(
&mut self,
accept: impl Fn(SocketAddr) -> bool,
rec: Option<&SockRec>,
) -> Option<(D, u32)> {
let idx = self.rx.iter().position(|q| accept(q.dg.src()))?;
let q = self.rx.remove(idx)?;
if let Some(rec) = rec {
rec.consumed(q.dg.payload(), q.charge, q.dg.src());
}
Some((q.dg, q.drops))
}
#[cfg(not(windows))]
pub(crate) fn peek(
&mut self,
accept: impl Fn(SocketAddr) -> bool,
rec: Option<&SockRec>,
) -> Option<(D, u32)>
where
D: Clone,
{
self.land(rec);
self.peek_landed(accept)
}
#[cfg(not(windows))]
pub(crate) fn first_matching(
&mut self,
accept: impl Fn(SocketAddr) -> bool,
rec: Option<&SockRec>,
) -> Option<(&D, u32)> {
self.land(rec);
let q = self.rx.iter().find(|q| accept(q.dg.src()))?;
Some((&q.dg, q.drops))
}
pub(crate) fn peek_landed(&self, accept: impl Fn(SocketAddr) -> bool) -> Option<(D, u32)>
where
D: Clone,
{
let q = self.rx.iter().find(|q| accept(q.dg.src()))?;
Some((q.dg.clone(), q.drops))
}
pub(crate) fn has(
&mut self,
accept: impl Fn(SocketAddr) -> bool,
rec: Option<&SockRec>,
) -> bool {
self.land(rec);
self.rx.iter().any(|q| accept(q.dg.src()))
}
#[cfg(windows)]
pub(crate) fn first_matching_at(
&mut self,
accept: impl Fn(SocketAddr) -> bool,
rec: Option<&SockRec>,
cutoff: std::time::Duration,
) -> Option<&D> {
self.land_at(rec, cutoff);
self.rx
.iter()
.find(|queued| accept(queued.dg.src()))
.map(|queued| &queued.dg)
}
pub(crate) fn next_len(&mut self, rec: Option<&SockRec>) -> Option<usize> {
self.land(rec);
self.rx.front().map(|q| q.dg.payload())
}
#[cfg_attr(windows, allow(dead_code))]
pub(crate) fn landed(&mut self, rec: Option<&SockRec>) -> u64 {
self.land(rec);
self.landed
}
pub(crate) fn queued(&self) -> (usize, usize) {
self.rx
.iter()
.fold((0, 0), |(n, bytes), q| (n + 1, bytes + q.dg.payload()))
}
}
#[derive(Clone, Copy)]
pub(crate) struct SockBuf {
pub(crate) rcvbuf: i32,
pub(crate) sndbuf: i32,
#[cfg(target_os = "linux")]
pub(crate) rcv_locked: bool,
enforce: bool,
#[cfg(target_os = "linux")]
enforce_error: bool,
#[cfg(target_os = "linux")]
pub(crate) error_mem: usize,
sb_max: usize,
overshoot: bool,
skb: (usize, usize),
pub(crate) rx: RxAccount,
pub(crate) overflowed: u64,
pub(crate) wire_lost: u64,
pub(crate) drops: u32,
pub(crate) rxq_ovfl: bool,
}
fn as_opt(v: usize) -> i32 {
v.min(i32::MAX as usize) as i32
}
impl SockBuf {
pub(crate) fn new(kind: SocketKind, limits: &SysLimits) -> Self {
let tcp = matches!(kind, SocketKind::TcpStream | SocketKind::TcpListener);
let (rcvbuf, sndbuf) = if tcp {
(limits.tcp_rmem_default, limits.tcp_wmem_default)
} else {
(limits.rmem_default, limits.wmem_default)
};
SockBuf {
rcvbuf: as_opt(rcvbuf),
sndbuf: as_opt(sndbuf),
#[cfg(target_os = "linux")]
rcv_locked: false,
enforce: limits.enforce_rcvbuf && kind == SocketKind::Udp,
#[cfg(target_os = "linux")]
enforce_error: limits.enforce_rcvbuf,
#[cfg(target_os = "linux")]
error_mem: 0,
sb_max: limits.rmem_max,
overshoot: limits.udp_rcvbuf_overshoot,
skb: (limits.skb_small_truesize, limits.skb_head_overhead),
rx: RxAccount::default(),
overflowed: 0,
wire_lost: 0,
drops: 0,
rxq_ovfl: false,
}
}
pub(crate) fn set(&mut self, which: Buf, value: i32) {
match which {
Buf::Rcv => {
self.rcvbuf = value;
#[cfg(target_os = "linux")]
{
self.rcv_locked = true;
}
}
Buf::Snd => self.sndbuf = value,
}
}
pub(crate) fn inherit(&mut self, listener: &SockBuf) {
self.rcvbuf = listener.rcvbuf;
self.sndbuf = listener.sndbuf;
#[cfg(target_os = "linux")]
{
self.rcv_locked = listener.rcv_locked;
}
}
pub(crate) fn tcp_send_space(&self) -> usize {
self.sndbuf.max(0) as usize
}
pub(crate) fn tcp_recv_space(&self) -> usize {
let rcvbuf = self.rcvbuf.max(0) as usize;
if cfg!(windows) {
rcvbuf.saturating_mul(2)
} else {
rcvbuf
}
}
pub(crate) fn admit(
&mut self,
len: usize,
v6: bool,
mtu: Option<u32>,
) -> Option<(Charge, u32)> {
let charge = self.rx.charge(len, v6, self.skb, mtu);
if self.enforce
&& !self
.rx
.fits(self.rcvbuf, self.sb_max, self.overshoot, charge)
{
self.overflowed += 1;
self.drops = self.drops.wrapping_add(1);
return None;
}
self.rx.take(len, charge);
Some((charge, if self.rxq_ovfl { self.drops } else { 0 }))
}
pub(crate) fn consumed(&mut self, len: usize, charge: Charge) {
self.rx.give_back(len, charge);
}
#[cfg(target_os = "linux")]
pub(crate) fn error_charge(&self, len: usize, v6: bool) -> usize {
linux_truesize(len, v6, self.skb)
}
#[cfg(target_os = "linux")]
pub(crate) fn take_tcp(&mut self, len: usize) -> usize {
let charge = self.skb.0.saturating_add(len);
self.rx.rmem_alloc = self.rx.rmem_alloc.saturating_add(charge);
charge
}
#[cfg(target_os = "linux")]
pub(crate) fn consume_tcp(&mut self, charge: usize) {
self.rx.rmem_alloc = self.rx.rmem_alloc.saturating_sub(charge);
}
#[cfg(target_os = "linux")]
pub(crate) fn admit_error(&mut self, charge: usize) -> bool {
if self.enforce_error
&& self.rx.rmem_alloc.saturating_add(charge) >= self.rcvbuf.max(0) as usize
{
return false;
}
self.rx.rmem_alloc += charge;
self.error_mem += charge;
true
}
#[cfg(target_os = "linux")]
pub(crate) fn consume_error(&mut self, charge: usize) {
self.rx.rmem_alloc = self.rx.rmem_alloc.saturating_sub(charge);
self.error_mem = self.error_mem.saturating_sub(charge);
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Buf {
Rcv,
Snd,
}
#[allow(unused_variables)]
pub(crate) fn sockbuf_value(
limits: &SysLimits,
buf: Buf,
val: c_int,
current: i32,
forced: bool,
) -> Result<i32, c_int> {
#[cfg(target_os = "linux")]
{
const SOCK_MIN_RCVBUF: i64 = 2304;
const SOCK_MIN_SNDBUF: i64 = 4608;
let (max, floor) = match buf {
Buf::Rcv => (limits.rmem_max, SOCK_MIN_RCVBUF),
Buf::Snd => (limits.wmem_max, SOCK_MIN_SNDBUF),
};
let v = if forced {
val.max(0) as u32
} else {
(val as u32).min(max.min(u32::MAX as usize) as u32)
};
let v = (v as i64).min(i32::MAX as i64 / 2);
Ok((v * 2).max(floor) as i32)
}
#[cfg(target_os = "macos")]
{
if val <= 0 {
return Err(libc::EINVAL);
}
if val as usize >= limits.sockbuf_reject_at {
return Err(libc::ENOBUFS);
}
let max = match buf {
Buf::Rcv => limits.rmem_max,
Buf::Snd => limits.wmem_max,
};
if val as usize > max && current as usize >= max {
return Err(libc::ENOBUFS);
}
Ok(as_opt((val as usize).min(max)))
}
#[cfg(windows)]
Ok(val)
}
#[cfg(target_os = "linux")]
pub(crate) const SO_RXQ_OVFL: c_int = 40;
#[cfg(unix)]
pub(crate) mod sockopt {
use std::ffi::c_int;
use super::{Buf, CAP_NET_ADMIN, CAP_NET_RAW, sockbuf_value};
use crate::scope::SimShared;
use crate::sockets::SockRec;
#[cfg(target_os = "linux")]
mod name {
use std::ffi::c_int;
pub(super) const SO_SNDBUFFORCE: c_int = 32;
pub(super) const SO_RCVBUFFORCE: c_int = 33;
pub(super) const SO_PRIORITY: c_int = 12;
pub(super) const SO_MARK: c_int = 36;
pub(super) const SO_MEMINFO: c_int = 55;
pub(super) const SO_COOKIE: c_int = 57;
pub(super) const SO_PROTOCOL: c_int = 38;
pub(super) const SO_DOMAIN: c_int = 39;
pub(super) const SO_BUSY_POLL: c_int = 46;
pub(super) const SO_PREFER_BUSY_POLL: c_int = 69;
pub(super) const SO_BUSY_POLL_BUDGET: c_int = 70;
}
fn read_int(val: *const u8, len: u32) -> Result<c_int, c_int> {
if val.is_null() || (len as usize) < size_of::<c_int>() {
return Err(libc::EINVAL);
}
Ok(unsafe { val.cast::<c_int>().read_unaligned() })
}
pub(crate) unsafe fn set(
shared: &SimShared,
rec: &SockRec,
level: c_int,
name: c_int,
val: *const u8,
len: u32,
) -> Option<Result<(), c_int>> {
if level != libc::SOL_SOCKET {
return None;
}
let buf = match name {
libc::SO_RCVBUF => Some((Buf::Rcv, false)),
libc::SO_SNDBUF => Some((Buf::Snd, false)),
#[cfg(target_os = "linux")]
name::SO_RCVBUFFORCE => Some((Buf::Rcv, true)),
#[cfg(target_os = "linux")]
name::SO_SNDBUFFORCE => Some((Buf::Snd, true)),
_ => None,
};
if let Some((which, forced)) = buf {
return Some((|| {
let v = read_int(val, len)?;
if forced && !shared.sys.has_cap(CAP_NET_ADMIN) {
return Err(libc::EPERM);
}
let mut state = rec.state();
let current = match which {
Buf::Rcv => state.buf.rcvbuf,
Buf::Snd => state.buf.sndbuf,
};
let stored = sockbuf_value(&shared.sys.limits(), which, v, current, forced)?;
state.buf.set(which, stored);
Ok(())
})());
}
#[cfg(target_os = "linux")]
{
let raw_or_admin =
|| shared.sys.has_cap(CAP_NET_RAW) || shared.sys.has_cap(CAP_NET_ADMIN);
match name {
name::SO_PRIORITY => {
return Some(read_int(val, len).and_then(|v| {
if !(0..=6).contains(&v) && !raw_or_admin() {
return Err(libc::EPERM);
}
rec.state().opts.priority = v;
Ok(())
}));
}
name::SO_MARK => {
return Some(read_int(val, len).and_then(|v| {
if !raw_or_admin() {
return Err(libc::EPERM);
}
rec.state().opts.mark = v as u32;
Ok(())
}));
}
super::SO_RXQ_OVFL => {
return Some(read_int(val, len).map(|v| rec.state().buf.rxq_ovfl = v != 0));
}
libc::SO_TYPE | name::SO_DOMAIN | name::SO_PROTOCOL => {
return Some(Err(libc::ENOPROTOOPT));
}
name::SO_BUSY_POLL => {
return Some(read_int(val, len).and_then(|v| {
if v < 0 {
return Err(libc::EINVAL);
}
rec.state().opts.busy_poll.usecs = v;
Ok(())
}));
}
name::SO_PREFER_BUSY_POLL => {
return Some(read_int(val, len).and_then(|v| {
if v != 0 && !shared.sys.has_cap(CAP_NET_ADMIN) {
return Err(libc::EPERM);
}
rec.state().opts.busy_poll.prefer = v != 0;
Ok(())
}));
}
name::SO_BUSY_POLL_BUDGET => {
return Some(read_int(val, len).and_then(|v| {
let mut state = rec.state();
if v > c_int::from(state.opts.busy_poll.budget)
&& !shared.sys.has_cap(CAP_NET_ADMIN)
{
return Err(libc::EPERM);
}
state.opts.busy_poll.budget = u16::try_from(v).map_err(|_| libc::EINVAL)?;
Ok(())
}));
}
_ => {}
}
}
let _ = (CAP_NET_RAW, shared);
None
}
unsafe fn write_bytes(bytes: &[u8], val: *mut u8, len: *mut u32) -> Result<(), c_int> {
if val.is_null() || len.is_null() {
return Err(libc::EFAULT);
}
let n = (unsafe { *len } as usize).min(bytes.len());
unsafe {
std::ptr::copy_nonoverlapping(bytes.as_ptr(), val, n);
*len = n as u32;
}
Ok(())
}
pub(crate) unsafe fn get(
rec: &SockRec,
level: c_int,
name: c_int,
val: *mut u8,
len: *mut u32,
) -> Option<Result<(), c_int>> {
if level != libc::SOL_SOCKET {
return None;
}
let int = |v: c_int| unsafe { write_bytes(&v.to_ne_bytes(), val, len) };
match name {
libc::SO_RCVBUF => return Some(int(rec.state().buf.rcvbuf)),
libc::SO_SNDBUF => return Some(int(rec.state().buf.sndbuf)),
_ => {}
}
#[cfg(target_os = "linux")]
match name {
name::SO_PRIORITY => return Some(int(rec.state().opts.priority)),
name::SO_MARK => return Some(int(rec.state().opts.mark as c_int)),
libc::SO_TYPE => {
let ty = rec.state().ty;
if ty != 0 {
return Some(int(ty));
}
}
name::SO_DOMAIN => return Some(int(rec.state().family.0)),
name::SO_PROTOCOL => return Some(int(rec.state().family.1)),
name::SO_BUSY_POLL => return Some(int(rec.state().opts.busy_poll.usecs)),
name::SO_PREFER_BUSY_POLL => {
return Some(int(rec.state().opts.busy_poll.prefer as c_int));
}
name::SO_BUSY_POLL_BUDGET => return Some(Err(libc::ENOPROTOOPT)),
super::SO_RXQ_OVFL => return Some(int(rec.state().buf.rxq_ovfl as c_int)),
name::SO_COOKIE => {
if len.is_null() || (unsafe { *len } as usize) < size_of::<u64>() {
return Some(Err(libc::EINVAL));
}
return Some(unsafe { write_bytes(&rec.id.get().to_ne_bytes(), val, len) });
}
name::SO_MEMINFO => {
rec.land();
let buf = rec.state().buf;
let rmem = buf.rx.rmem_alloc as u32;
let ordinary = rmem.saturating_sub(buf.error_mem as u32);
let words = [
rmem,
buf.rcvbuf as u32,
0,
buf.sndbuf as u32,
(4096 - ordinary % 4096) % 4096,
0,
0,
0,
buf.drops,
];
let bytes: Vec<u8> = words.iter().flat_map(|w| w.to_ne_bytes()).collect();
return Some(unsafe { write_bytes(&bytes, val, len) });
}
_ => {}
}
#[cfg(target_os = "macos")]
match name {
libc::SO_NREAD => return Some(int(rec.nread())),
libc::SO_NWRITE => return Some(int(0)),
_ => {}
}
None
}
}