use std::collections::{BTreeSet, HashMap, VecDeque};
use std::ffi::c_int;
use std::io;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use snare_interpose::{Fs, Net, NetResult};
#[cfg(target_os = "macos")]
mod mac_shutdown;
#[derive(Clone)]
enum Payload {
Inline { bytes: [u8; 128], len: u8 },
Heap(Vec<u8>),
}
impl Payload {
fn copy_from(data: &[u8]) -> Self {
if data.len() <= 128 {
let mut bytes = [0; 128];
bytes[..data.len()].copy_from_slice(data);
Self::Inline {
bytes,
len: data.len() as u8,
}
} else {
Self::Heap(data.to_vec())
}
}
fn into_vec(self) -> Vec<u8> {
match self {
Self::Inline { bytes, len } => bytes[..len as usize].to_vec(),
Self::Heap(data) => data,
}
}
}
impl std::ops::Deref for Payload {
type Target = [u8];
fn deref(&self) -> &Self::Target {
match self {
Self::Inline { bytes, len } => &bytes[..*len as usize],
Self::Heap(data) => data,
}
}
}
#[derive(Clone)]
pub(crate) struct Datagram {
src: SocketAddr,
data: Payload,
arrives: Option<Deadline>,
via: Option<(Arc<LinkState>, u64)>,
stamp: Stamp,
#[cfg(target_os = "linux")]
rx_fallback: Arc<std::sync::OnceLock<Duration>>,
mtu: Option<u32>,
sequence: u64,
}
impl Arrival for Datagram {
fn src(&self) -> SocketAddr {
self.src
}
fn payload(&self) -> usize {
self.data.len()
}
fn arrives(&self) -> Option<Deadline> {
self.arrives
}
fn lost(&self) -> bool {
match (&self.via, self.arrives) {
(Some((link, epoch)), Some(at)) => link.lost_in_flight(*epoch, at.instant()),
_ => false,
}
}
fn path_mtu(&self) -> Option<u32> {
self.mtu
}
}
#[derive(Default)]
pub(crate) struct DgramQueue {
packets: Mutex<RxQueue<Datagram>>,
rec: Option<Arc<SockRec>>,
next_port: AtomicU64,
}
impl DgramQueue {
fn for_socket(rec: &Arc<SockRec>) -> Arc<Self> {
let queue = Arc::new(DgramQueue {
packets: Mutex::default(),
rec: Some(rec.clone()),
next_port: AtomicU64::new(0),
});
rec.set_probe(Arc::downgrade(&queue) as std::sync::Weak<dyn RxProbe>);
queue
}
#[allow(clippy::too_many_arguments)]
fn push_after(
&self,
src: SocketAddr,
data: &[u8],
delay: Duration,
via: Option<(Arc<LinkState>, u64)>,
sent: Stamp,
mtu: Option<u32>,
domain: usize,
) {
let arrives = (!delay.is_zero()).then(|| Deadline::after(delay));
if let Some(arrives) = arrives {
arrives.wake_waiters_then();
}
let dg = Datagram {
src,
data: Payload::copy_from(data),
arrives,
via,
stamp: sent.later(delay),
#[cfg(target_os = "linux")]
rx_fallback: Arc::default(),
mtu,
sequence: readiness_sequence(),
};
self.packets.lock().unwrap().push(dg, self.rec.as_deref());
if let Some(rec) = &self.rec {
readiness().bump_keys(domain, &[rec.wake_key()]);
} else {
readiness().bump_keys(domain, &[]);
}
}
fn pop(&self, accept: impl Fn(SocketAddr) -> bool) -> Option<(Datagram, u32)> {
self.packets
.lock()
.unwrap()
.pop(accept, self.rec.as_deref())
}
fn peek(&self, accept: impl Fn(SocketAddr) -> bool) -> Option<(Datagram, u32)> {
self.packets
.lock()
.unwrap()
.peek(accept, self.rec.as_deref())
}
fn readiness_metadata(&self) -> Option<(Duration, u64, usize)> {
self.packets
.lock()
.unwrap()
.first_matching(|_| true, self.rec.as_deref())
.map(|(d, _)| (d.stamp.mono, d.sequence, d.data.len()))
}
fn has(&self, accept: impl Fn(SocketAddr) -> bool) -> bool {
self.packets
.lock()
.unwrap()
.has(accept, self.rec.as_deref())
}
fn landed(&self) -> u64 {
self.packets.lock().unwrap().landed(self.rec.as_deref())
}
}
impl RxProbe for DgramQueue {
fn pending_time(&self) -> bool {
self.packets.lock().unwrap().pending_time()
}
fn land(&self) {
self.packets.lock().unwrap().land(self.rec.as_deref());
}
fn queued(&self) -> (usize, usize) {
self.packets.lock().unwrap().queued()
}
fn next_len(&self) -> Option<usize> {
self.packets.lock().unwrap().next_len(self.rec.as_deref())
}
}
pub(crate) struct Conn {
a_to_b: Arc<Pipe>,
b_to_a: Arc<Pipe>,
pub(crate) client: SocketAddr,
pub(crate) server: SocketAddr,
regs: Arc<Registries>,
hop: Option<Hop>,
pair: bool,
tap: Option<TcpTap>,
stamps: [Mutex<VecDeque<(usize, Stamp)>>; 2],
write_delays: [AtomicU64; 2],
#[cfg(target_os = "macos")]
mac_shutdown: Mutex<mac_shutdown::State>,
}
#[derive(Clone, Copy, PartialEq)]
enum End {
A,
B,
}
impl Conn {
fn write(&self, end: End, bytes: &[u8]) -> Option<usize> {
#[cfg(target_os = "macos")]
self.progress_tcp_shutdown();
let pipe = self.write_pipe(end);
if (pipe.reader_gone()
|| (cfg!(not(target_os = "macos"))
&& pipe.is_read_shut()
&& self.read_pipe(end).is_closed()))
&& !pipe.is_closed()
{
self.reset(if end == End::A { End::B } else { End::A });
return Some(bytes.len());
}
let delay = self.delay();
self.write_delays[end.index()].store(
delay.as_nanos().min(u64::MAX as u128) as u64,
Ordering::Relaxed,
);
#[cfg(target_os = "macos")]
let mut shutdown_at = None;
let written =
self.write_pipe(end)
.write_after(bytes, delay, |sent, arrival, _read_shutdown_at| {
self.record_write(end, sent, arrival, None, _read_shutdown_at.is_none());
#[cfg(target_os = "macos")]
if !sent.is_empty() {
shutdown_at = _read_shutdown_at;
}
});
#[cfg(target_os = "macos")]
if let Some(at) = shutdown_at {
self.schedule_read_shutdown_data(end, at);
}
written
}
fn record_write(
&self,
end: End,
bytes: &[u8],
arrival: Duration,
mss: Option<usize>,
_acknowledge: bool,
) {
let shared = &self.regs.shared;
if let Some(tap) = &self.tap {
#[cfg(target_os = "macos")]
if !_acknowledge {
tap.data_mss_unacknowledged(end.index(), bytes, arrival, mss);
} else {
tap.data_mss(end.index(), bytes, arrival, mss);
}
#[cfg(not(target_os = "macos"))]
tap.data_mss(end.index(), bytes, arrival, mss);
}
self.stamps[end.index()]
.lock()
.unwrap()
.push_back((bytes.len(), shared.tstamp_now().later(arrival)));
if !self.pair {
let pipe = self.write_pipe(end);
shared.account_tcp_mss(
self.hop.as_ref(),
end == End::A,
bytes.len(),
self.server.ip(),
(pipe.has_writer(), pipe.has_reader()),
mss,
);
}
}
fn delay(&self) -> Duration {
if self.pair {
return Duration::ZERO;
}
let shared = &self.regs.shared;
shared.policies.tcp_delay(self.server, self.client) + shared.hop_delay(self.hop.as_ref())
}
fn took(&self, end: End, n: usize) -> Option<Stamp> {
let writer = match end {
End::A => End::B,
End::B => End::A,
};
let mut chunks = self.stamps[writer.index()].lock().unwrap();
let mut left = n;
let mut last = None;
while left > 0 {
let Some((len, at)) = chunks.front_mut() else {
break;
};
last = Some(*at);
if *len <= left {
left -= *len;
chunks.pop_front();
} else {
*len -= left;
left = 0;
}
}
last
}
fn read_pipe(&self, end: End) -> &Pipe {
match end {
End::A => &self.b_to_a,
End::B => &self.a_to_b,
}
}
fn write_pipe(&self, end: End) -> &Arc<Pipe> {
match end {
End::A => &self.a_to_b,
End::B => &self.b_to_a,
}
}
fn close(&self, end: End) {
#[cfg(target_os = "macos")]
self.progress_tcp_shutdown();
self.write_pipe(end).close_writer();
self.write_pipe(end).close(|arrival| {
if let Some(tap) = &self.tap {
tap.fin(end.index(), arrival);
}
});
}
fn local(&self, end: End) -> SocketAddr {
match end {
End::A => self.client,
End::B => self.server,
}
}
fn peer(&self, end: End) -> SocketAddr {
match end {
End::A => self.server,
End::B => self.client,
}
}
fn reset(&self, from: End) {
if !self.a_to_b.is_reset() && !self.pair {
self.regs.shared.count_reset(
self.server.ip(),
self.write_pipe(from).has_writer(),
[
self.a_to_b.established_reader(),
self.b_to_a.established_reader(),
],
);
}
if let Some(tap) = &self.tap {
tap.rst(from.index());
}
if cfg!(target_os = "linux") {
for (incoming, outgoing) in [(&self.a_to_b, &self.b_to_a), (&self.b_to_a, &self.a_to_b)]
{
if if incoming.is_read_shut() {
outgoing.is_write_shut()
} else {
!incoming.is_closed()
} {
incoming.reset_reader();
}
}
} else {
self.write_pipe(from).reset_reader();
}
self.a_to_b.reset();
self.b_to_a.reset();
}
}
impl End {
fn index(self) -> usize {
match self {
End::A => 0,
End::B => 1,
}
}
}
pub struct Listener {
rec: Option<std::sync::Weak<SockRec>>,
addr: SocketAddr,
pending: Mutex<VecDeque<Arc<Conn>>>,
capacity: AtomicUsize,
occupied: AtomicUsize,
tester: bool,
arrivals: AtomicU64,
ready_order: AtomicU64,
ready_at: AtomicU64,
}
impl Listener {
fn wake_key(&self) -> Option<crate::readiness::WakeKey> {
self.rec.as_ref()?.upgrade().map(|rec| rec.wake_key())
}
pub(crate) fn try_accept(&self) -> Option<Arc<Conn>> {
snare_interpose::real(|| {
let conn = self.pending.lock().unwrap().pop_front()?;
self.occupied.fetch_sub(1, Ordering::AcqRel);
Some(conn)
})
}
pub(crate) fn has_pending(&self) -> bool {
snare_interpose::real(|| !self.pending.lock().unwrap().is_empty())
}
fn reserve(self: &Arc<Self>) -> Option<ListenerAdmission> {
let mut used = self.occupied.load(Ordering::Acquire);
loop {
if used >= self.capacity.load(Ordering::Acquire) {
return None;
}
match self.occupied.compare_exchange_weak(
used,
used + 1,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => {
return Some(ListenerAdmission {
listener: self.clone(),
reserved: true,
});
}
Err(current) => used = current,
}
}
}
fn queue(&self, conn: Arc<Conn>) {
self.ready_at.store(
conn.regs
.shared
.tstamp_now()
.mono
.as_nanos()
.min(u64::MAX as u128) as u64,
Ordering::Relaxed,
);
self.pending.lock().unwrap().push_back(conn);
self.arrivals.fetch_add(1, Ordering::Relaxed);
self.ready_order
.store(readiness_sequence(), Ordering::Relaxed);
}
}
struct ListenerAdmission {
listener: Arc<Listener>,
reserved: bool,
}
impl ListenerAdmission {
fn queue(mut self, conn: Arc<Conn>) {
self.listener.queue(conn);
self.reserved = false;
}
}
impl Drop for ListenerAdmission {
fn drop(&mut self) {
if self.reserved {
self.listener.occupied.fetch_sub(1, Ordering::AcqRel);
}
}
}
pub(crate) struct Registries {
listeners: Mutex<HashMap<SocketAddr, Arc<Listener>>>,
udp: Mutex<UdpRegistry>,
foreign_udp: Mutex<Vec<std::sync::Weak<dyn ForeignUdp>>>,
pub(crate) shared: Arc<SimShared>,
}
impl Registries {
fn new(shared: Arc<SimShared>) -> Self {
Registries {
listeners: Mutex::default(),
udp: Mutex::default(),
foreign_udp: Mutex::default(),
shared,
}
}
fn deliver(
&self,
sender: &Sender,
src: SocketAddr,
dest: SocketAddr,
data: &[u8],
tx: bool,
) -> usize {
let sent = self.shared.tstamp_now();
let joined = dest.ip().is_multicast() && self.shared.host_joined(dest.ip());
let cands = self.udp.lock().unwrap().delivery_candidates(dest, joined);
let station = match &cands {
UdpCandidates::One(None) => false,
UdpCandidates::One(Some(cand)) if cand.addr == dest => false,
_ => self.station_at(dest.ip()),
};
let wire = Wire {
src,
dest,
len: data.len(),
};
let emit = |copy: crate::netif::Copy<Arc<DgramQueue>>| {
if let Some(rec) = ©.q.rec {
if copy.delays.is_empty() {
rec.count_wire_lost();
} else {
rec.note_rx_nic(copy.nic.as_deref());
}
}
for delay in copy.delays {
copy.q.push_after(
src,
data,
delay,
copy.via.clone(),
sent,
copy.mtu,
self.shared.domain_key(),
);
}
};
match cands {
UdpCandidates::One(cand) => {
if let Some(copy) = self.shared.fan_out_one(sender, wire, station, cand, tx) {
emit(copy);
1
} else {
0
}
}
UdpCandidates::Many(cands) => {
let copies = self.shared.fan_out(sender, wire, station, cands, tx);
let reached = copies.len();
for copy in copies {
emit(copy);
}
reached
}
}
}
pub(crate) fn station_at(&self, ip: IpAddr) -> bool {
let endpoint = self.udp.lock().unwrap().endpoints.contains_key(&ip);
endpoint
|| self
.listeners
.lock()
.unwrap()
.values()
.any(|l| l.tester && l.addr.ip() == ip)
}
}
pub(crate) trait ForeignUdp: Send + Sync {
fn deliver_from_peer(&self, src: SocketAddr, dest: SocketAddr, data: &[u8]) -> usize;
}
pub(crate) fn attach_foreign_udp(regs: &Registries, backend: std::sync::Weak<dyn ForeignUdp>) {
snare_interpose::real(|| regs.foreign_udp.lock().unwrap().push(backend));
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub(crate) fn deliver_to_endpoints(
regs: &Registries,
sender: &Sender,
src: SocketAddr,
dest: SocketAddr,
data: &[u8],
) -> usize {
snare_interpose::real(|| regs.deliver(sender, src, dest, data, false))
}
#[derive(Default)]
struct UdpRegistry {
bound: HashMap<SocketAddr, Arc<DgramQueue>>,
by_port: HashMap<u16, UdpBindings>,
endpoints: HashMap<IpAddr, usize>,
ports: HashMap<IpAddr, UdpPorts>,
families: [UdpPorts; 2],
references: HashMap<(bool, u16), usize>,
addresses: HashMap<(IpAddr, u16), usize>,
}
enum UdpCandidates {
One(Option<Cand<Arc<DgramQueue>>>),
Many(Vec<Cand<Arc<DgramQueue>>>),
}
struct UdpBindings {
first: (SocketAddr, Arc<DgramQueue>),
rest: Vec<(SocketAddr, Arc<DgramQueue>)>,
}
#[derive(Default)]
struct UdpPorts {
words: Vec<u64>,
first: usize,
count: usize,
}
impl UdpPorts {
const FIRST: u16 = 49152;
const WORDS: usize = 256;
fn insert(&mut self, port: u16) {
let bit = usize::from(port - Self::FIRST);
if self.words.len() <= bit / 64 {
self.words.resize(bit / 64 + 1, 0);
}
self.words[bit / 64] |= 1 << (bit % 64);
self.count += 1;
while self.first < self.words.len() && self.words[self.first] == u64::MAX {
self.first += 1;
}
}
fn remove(&mut self, port: u16) {
let bit = usize::from(port - Self::FIRST);
self.words[bit / 64] &= !(1 << (bit % 64));
self.first = self.first.min(bit / 64);
self.count -= 1;
}
fn find(&self, start: usize, end: usize) -> Option<u16> {
if self.words.is_empty() {
return (start < end).then_some(Self::FIRST + start as u16);
}
let mut word = (start / 64).max(self.first);
while word < Self::WORDS && word * 64 < end {
let low = start.saturating_sub(word * 64);
let high = (end - word * 64).min(64);
let mask = (u64::MAX << low)
& if high == 64 {
u64::MAX
} else {
(1 << high) - 1
};
let free = !self.words.get(word).copied().unwrap_or(0) & mask;
if free != 0 {
return Some(Self::FIRST + (word * 64 + free.trailing_zeros() as usize) as u16);
}
word += 1;
}
None
}
fn alloc_from(&self, first: u16) -> u16 {
let start = usize::from(first.max(Self::FIRST) - Self::FIRST);
self.find(start, Self::WORDS * 64)
.or_else(|| self.find(0, start))
.unwrap_or(0)
}
}
impl UdpRegistry {
fn add_endpoint(&mut self, ip: IpAddr) {
*self.endpoints.entry(ip).or_default() += 1;
}
fn remove_endpoint(&mut self, ip: IpAddr) {
let count = self.endpoints.get_mut(&ip).unwrap();
*count -= 1;
if *count == 0 {
self.endpoints.remove(&ip);
}
}
fn insert(&mut self, addr: SocketAddr, queue: Arc<DgramQueue>) {
let endpoint = queue.rec.is_none();
let indexed = queue.clone();
if let Some(previous) = self.bound.insert(addr, queue) {
let bucket = self.by_port.get_mut(&addr.port()).unwrap();
let entry = if bucket.first.0 == addr {
&mut bucket.first
} else {
bucket
.rest
.iter_mut()
.find(|entry| entry.0 == addr)
.unwrap()
};
entry.1 = indexed;
if previous.rec.is_none() != endpoint {
if endpoint {
self.add_endpoint(addr.ip());
} else {
self.remove_endpoint(addr.ip());
}
}
return;
}
if endpoint {
self.add_endpoint(addr.ip());
}
match self.by_port.entry(addr.port()) {
std::collections::hash_map::Entry::Occupied(mut bucket) => {
bucket.get_mut().rest.push((addr, indexed));
}
std::collections::hash_map::Entry::Vacant(bucket) => {
bucket.insert(UdpBindings {
first: (addr, indexed),
rest: Vec::new(),
});
}
}
if addr.port() < UdpPorts::FIRST {
return;
}
let addresses = self.addresses.entry((addr.ip(), addr.port())).or_default();
if *addresses == 0 {
self.ports.entry(addr.ip()).or_default().insert(addr.port());
}
*addresses += 1;
let references = self
.references
.entry((addr.is_ipv4(), addr.port()))
.or_default();
if *references == 0 {
self.families[usize::from(addr.is_ipv4())].insert(addr.port());
}
*references += 1;
}
fn remove(&mut self, addr: &SocketAddr) -> Option<Arc<DgramQueue>> {
let queue = self.bound.remove(addr)?;
if queue.rec.is_none() {
self.remove_endpoint(addr.ip());
}
let bucket = self.by_port.get_mut(&addr.port()).unwrap();
if bucket.first.0 == *addr {
if let Some(last) = bucket.rest.pop() {
bucket.first = last;
} else {
self.by_port.remove(&addr.port());
}
} else {
let index = bucket
.rest
.iter()
.position(|bound| bound.0 == *addr)
.unwrap();
bucket.rest.swap_remove(index);
}
if addr.port() >= UdpPorts::FIRST {
let address = (addr.ip(), addr.port());
let addresses = self.addresses.get_mut(&address).unwrap();
*addresses -= 1;
if *addresses == 0 {
self.addresses.remove(&address);
let ports = self.ports.get_mut(&addr.ip()).unwrap();
ports.remove(addr.port());
if ports.count == 0 {
self.ports.remove(&addr.ip());
}
}
let key = (addr.is_ipv4(), addr.port());
let references = self.references.get_mut(&key).unwrap();
*references -= 1;
if *references == 0 {
self.references.remove(&key);
self.families[usize::from(addr.is_ipv4())].remove(addr.port());
}
}
Some(queue)
}
fn alloc_port(&self, ip: IpAddr) -> u16 {
self.alloc_port_from(ip, 49152)
}
fn alloc_port_from(&self, ip: IpAddr, first: u16) -> u16 {
if ip.is_unspecified() {
self.families[usize::from(ip.is_ipv4())].alloc_from(first)
} else {
self.ports
.get(&ip)
.map_or(first.max(UdpPorts::FIRST), |ports| ports.alloc_from(first))
}
}
fn delivery_candidates(&self, dest: SocketAddr, joined: bool) -> UdpCandidates {
let Some(bucket) = self.by_port.get(&dest.port()) else {
return UdpCandidates::One(None);
};
if !bucket.rest.is_empty() {
return UdpCandidates::Many(self.candidates(dest, joined));
}
let (addr, queue) = &bucket.first;
let group = dest.ip();
let accepted = !group.is_multicast()
|| queue
.rec
.as_ref()
.is_some_and(|rec| rec.takes_group(*addr, group, joined));
UdpCandidates::One(accepted.then(|| Self::candidate(*addr, queue)))
}
fn candidate(addr: SocketAddr, queue: &Arc<DgramQueue>) -> Cand<Arc<DgramQueue>> {
Cand {
addr,
endpoint: queue.rec.is_none(),
device: queue
.rec
.as_ref()
.and_then(|rec| rec.state().device.as_ref().map(|device| device.0)),
q: queue.clone(),
}
}
fn candidates(&self, dest: SocketAddr, joined: bool) -> Vec<Cand<Arc<DgramQueue>>> {
let Some(bucket) = self.by_port.get(&dest.port()) else {
return Vec::new();
};
let group = dest.ip();
std::iter::once(&bucket.first)
.chain(bucket.rest.iter())
.filter(|(addr, q)| {
!group.is_multicast()
|| q.rec
.as_ref()
.is_some_and(|r| r.takes_group(*addr, group, joined))
})
.map(|(addr, q)| Self::candidate(*addr, q))
.collect()
}
}
fn unspecified_like(ip: IpAddr) -> IpAddr {
match ip {
IpAddr::V4(_) => IpAddr::V4(Ipv4Addr::UNSPECIFIED),
IpAddr::V6(_) => IpAddr::V6(Ipv6Addr::UNSPECIFIED),
}
}
fn unhashed(requested: Option<SocketAddr>) -> Option<SocketAddr> {
requested.map(|r| SocketAddr::new(r.ip(), 0))
}
thread_local! {
static CURRENT: std::cell::RefCell<Option<Arc<Registries>>> = const { std::cell::RefCell::new(None) };
}
pub(crate) fn enter(regs: Arc<Registries>) -> RegistryGuard {
let previous = CURRENT.with(|c| c.replace(Some(regs)));
RegistryGuard { previous }
}
pub(crate) struct RegistryGuard {
previous: Option<Arc<Registries>>,
}
impl Drop for RegistryGuard {
fn drop(&mut self) {
CURRENT.with(|c| *c.borrow_mut() = self.previous.take());
}
}
fn current() -> Arc<Registries> {
try_registries_here().expect("snare tester functions must be called inside Sim::run")
}
pub(crate) fn try_registries_here() -> Option<Arc<Registries>> {
CURRENT.with(|c| c.borrow().clone())
}
pub(crate) fn registries_here() -> Arc<Registries> {
current()
}
pub(crate) struct ScopeLayer(pub(crate) Arc<Registries>);
impl snare_interpose::Layer for ScopeLayer {
fn thread_started(&self) {
std::mem::forget(enter(self.0.clone()));
}
}
pub(crate) fn unlisten(regs: &Registries, listener: &Arc<Listener>) {
snare_interpose::real(|| {
let mut listeners = regs.listeners.lock().unwrap();
if listeners
.get(&listener.addr)
.is_some_and(|l| Arc::ptr_eq(l, listener))
{
listeners.remove(&listener.addr);
}
});
}
pub(crate) struct UdpEndpoint {
regs: Arc<Registries>,
addr: SocketAddr,
queue: Arc<DgramQueue>,
}
impl UdpEndpoint {
pub(crate) fn bind(regs: Arc<Registries>, addr: SocketAddr) -> Self {
let queue = Arc::new(DgramQueue::default());
snare_interpose::real(|| {
let mut udp = regs.udp.lock().unwrap();
assert!(
!udp.bound.contains_key(&addr),
"udp tester address {addr} is already bound"
);
udp.insert(addr, queue.clone());
});
UdpEndpoint { regs, addr, queue }
}
pub(crate) fn try_recv(&self) -> Option<(SocketAddr, Vec<u8>)> {
snare_interpose::real(|| {
self.queue
.pop(|_| true)
.map(|(dg, _)| (dg.src, dg.data.into_vec()))
})
}
pub(crate) fn has_pending(&self) -> bool {
snare_interpose::real(|| self.queue.has(|_| true))
}
pub(crate) fn send_to(&self, dest: SocketAddr, data: &[u8]) {
snare_interpose::real(|| {
let sender = Sender::Station(self.addr.ip());
self.regs
.shared
.capture_udp(&sender, self.addr, dest, Dir::In, data, None);
let mut reached = self.regs.deliver(&sender, self.addr, dest, data, true);
let foreign: Vec<_> = self.regs.foreign_udp.lock().unwrap().clone();
for backend in foreign.iter().filter_map(std::sync::Weak::upgrade) {
reached += backend.deliver_from_peer(self.addr, dest, data);
}
if reached == 0 {
let station = self.regs.station_at(dest.ip());
self.regs.shared.count_unreceived(dest, station);
}
});
}
}
impl Drop for UdpEndpoint {
fn drop(&mut self) {
snare_interpose::real(|| {
let mut udp = self.regs.udp.lock().unwrap();
if udp
.bound
.get(&self.addr)
.is_some_and(|q| Arc::ptr_eq(q, &self.queue))
{
udp.remove(&self.addr);
}
});
}
}
pub(crate) fn listen_at(addr: SocketAddr) -> Arc<Listener> {
let listener = Arc::new(Listener {
rec: None,
addr,
pending: Mutex::default(),
capacity: AtomicUsize::new(usize::MAX),
occupied: AtomicUsize::new(0),
tester: true,
arrivals: AtomicU64::new(0),
ready_order: AtomicU64::new(0),
ready_at: AtomicU64::new(0),
});
snare_interpose::real(|| {
current()
.listeners
.lock()
.unwrap()
.insert(addr, listener.clone())
});
listener
}
pub(crate) use crate::readiness::readiness;
use crate::scope::SimShared;
use std::time::Duration;
use crate::faults::{ConnectAttempt, Outcome, Syn};
use crate::limits::{Arrival, RxQueue};
use crate::netif::{Cand, Hop, LinkState, Op, Sender, Wire};
use crate::pcapng::{Dir, TcpTap};
use crate::readiness::Deadline;
use crate::sockets::{Membership, RxProbe, SockRec, SocketKind, Taker, UnmodelledOption};
use crate::stream::{Ends, Pipe, Read, Sent};
use crate::tstamp::{Rx, Stamp};
enum Sock {
Fresh {
domain: c_int,
local: Option<SocketAddr>,
nonblocking: bool,
rec: Arc<SockRec>,
},
Connecting {
domain: c_int,
local: Option<SocketAddr>,
nonblocking: bool,
rec: Arc<SockRec>,
dest: SocketAddr,
attempt: Arc<Mutex<ConnectAttempt>>,
},
Stream {
conn: Arc<Conn>,
end: End,
nonblocking: bool,
rec: Arc<SockRec>,
},
Listener {
listener: Arc<Listener>,
nonblocking: bool,
rec: Arc<SockRec>,
},
Dgram {
queue: Arc<DgramQueue>,
domain: c_int,
local: Option<SocketAddr>,
requested: Option<SocketAddr>,
peer: Option<SocketAddr>,
nonblocking: bool,
broadcast: bool,
rec: Arc<SockRec>,
},
UnixDgram {
queue: Arc<DgramQueue>,
peer: std::sync::Weak<DgramQueue>,
nonblocking: bool,
rec: Arc<SockRec>,
},
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
Event {
ready_order: u64,
ready_at: Duration,
counter: u64,
nonblocking: bool,
semaphore: bool,
writes: u64,
reads: u64,
},
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
Epoll {
interests: HashMap<(c_int, EpollIdentity), EpollInterest>,
},
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), allow(dead_code))]
Raw {
ifindex: Option<u32>,
rx: VecDeque<Vec<u8>>,
arrived: u64,
nonblocking: bool,
rec: Arc<SockRec>,
},
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
Kqueue { state: Arc<Mutex<KqueueState>> },
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
enum EpollIdentity {
Local(u64),
Foreign(u64),
Untracked,
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
struct EpollInterest {
order: u64,
pending_order: Option<(Duration, u64)>,
events: u32,
data: u64,
edge: Edge,
spent: bool,
}
#[derive(Default)]
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
pub(crate) struct KqueueState {
reads: HashMap<c_int, Knote>,
writes: HashMap<c_int, Knote>,
users: HashMap<usize, UserNote>,
}
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
struct Knote {
udata: u64,
flags: u16,
disabled: bool,
edge: Edge,
}
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
struct UserNote {
udata: u64,
flags: u16,
disabled: bool,
triggered: bool,
}
struct SocketDescription {
id: u64,
sock: Sock,
status_flags: c_int,
descriptors: BTreeSet<c_int>,
}
#[derive(Clone, Copy, PartialEq, Eq)]
struct DescriptionKey {
slot: usize,
id: u64,
}
#[derive(Default)]
struct Sockets {
descriptions: Vec<Option<SocketDescription>>,
free_slots: Vec<usize>,
descriptors: HashMap<c_int, DescriptionKey>,
#[cfg(target_os = "linux")]
identities: HashMap<u64, usize>,
next_id: u64,
}
impl Sockets {
fn identity(&self, fd: c_int) -> Option<u64> {
self.descriptors.get(&fd).map(|key| key.id)
}
#[cfg(target_os = "linux")]
fn get_identity(&self, id: u64) -> Option<&Sock> {
let description = self
.descriptions
.get(*self.identities.get(&id)?)?
.as_ref()?;
(description.id == id).then_some(&description.sock)
}
fn get(&self, fd: &c_int) -> Option<&Sock> {
let key = self.descriptors.get(fd)?;
let description = self.descriptions.get(key.slot)?.as_ref()?;
(description.id == key.id).then_some(&description.sock)
}
fn get_mut(&mut self, fd: &c_int) -> Option<&mut Sock> {
Some(&mut self.description_mut(fd)?.sock)
}
fn description_mut(&mut self, fd: &c_int) -> Option<&mut SocketDescription> {
let key = self.descriptors.get(fd)?;
self.descriptions
.get_mut(key.slot)?
.as_mut()
.filter(|description| description.id == key.id)
}
fn contains_key(&self, fd: &c_int) -> bool {
self.descriptors.contains_key(fd)
}
fn insert(&mut self, fd: c_int, sock: Sock) {
if let Some(description) = self.description_mut(&fd) {
description.sock = sock;
} else {
self.next_id += 1;
let id = self.next_id;
let slot = self.free_slots.pop().unwrap_or_else(|| {
self.descriptions.push(None);
self.descriptions.len() - 1
});
self.descriptions[slot] = Some(SocketDescription {
id,
sock,
status_flags: libc::O_RDWR,
descriptors: BTreeSet::from([fd]),
});
self.descriptors.insert(fd, DescriptionKey { slot, id });
#[cfg(target_os = "linux")]
self.identities.insert(id, slot);
}
}
fn alias(&mut self, fd: c_int, new_fd: c_int) {
let key = self.descriptors[&fd];
self.descriptors.insert(new_fd, key);
self.descriptions[key.slot]
.as_mut()
.unwrap()
.descriptors
.insert(new_fd);
}
fn remove(&mut self, fd: &c_int) -> Option<Option<Sock>> {
let key = self.descriptors.remove(fd)?;
let description = self.descriptions[key.slot].as_mut().unwrap();
description.descriptors.remove(fd);
Some(if description.descriptors.is_empty() {
let retired = self.descriptions[key.slot].take().unwrap();
self.free_slots.push(key.slot);
#[cfg(target_os = "linux")]
self.identities.remove(&key.id);
Some(retired.sock)
} else {
None
})
}
fn values(&self) -> impl Iterator<Item = &Sock> {
self.descriptions
.iter()
.flatten()
.map(|description| &description.sock)
}
fn iter(&self) -> impl Iterator<Item = (&c_int, &Sock)> {
self.descriptors
.iter()
.map(|(fd, key)| (fd, &self.descriptions[key.slot].as_ref().unwrap().sock))
}
fn iter_mut(&mut self) -> impl Iterator<Item = (&c_int, &mut Sock)> {
self.descriptions.iter_mut().flatten().map(|description| {
(
description.descriptors.first().unwrap(),
&mut description.sock,
)
})
}
}
pub struct Fabric {
socks: Mutex<Sockets>,
regs: Arc<Registries>,
devnull: c_int,
}
struct UdpSendEndpoint {
local: Option<SocketAddr>,
requested: Option<SocketAddr>,
queue: Arc<DgramQueue>,
domain: c_int,
broadcast: bool,
nonblocking: bool,
rec: Arc<SockRec>,
}
impl UdpSendEndpoint {
fn from_sock(sock: &Sock) -> Option<Self> {
match sock {
Sock::Dgram {
local,
requested,
queue,
domain,
broadcast,
nonblocking,
rec,
..
} => Some(Self {
local: *local,
requested: *requested,
queue: queue.clone(),
domain: *domain,
broadcast: *broadcast,
nonblocking: *nonblocking,
rec: rec.clone(),
}),
_ => None,
}
}
}
struct DgramRecvEndpoint {
queue: Arc<DgramQueue>,
nonblocking: bool,
rec: Arc<SockRec>,
}
impl DgramRecvEndpoint {
fn from_sock(sock: &Sock) -> Option<Self> {
match sock {
Sock::Dgram {
queue,
nonblocking,
rec,
..
}
| Sock::UnixDgram {
queue,
nonblocking,
rec,
..
} => Some(Self {
queue: queue.clone(),
nonblocking: *nonblocking,
rec: rec.clone(),
}),
_ => None,
}
}
}
impl Fabric {
pub(crate) fn new(shared: Arc<SimShared>) -> Self {
let devnull = snare_interpose::real(|| unsafe {
libc::open(c"/dev/null".as_ptr(), libc::O_RDWR | libc::O_CLOEXEC)
});
Fabric {
socks: Mutex::new(Sockets::default()),
regs: Arc::new(Registries::new(shared)),
devnull,
}
}
fn detach_fd(&self, fd: c_int) -> Option<()> {
let (removed, rec, key) = {
let mut socks = self.socks.lock().unwrap();
let rec = socks.get(&fd)?.rec().cloned();
let key = rec.as_ref().map(|rec| rec.wake_key()).or_else(|| {
socks
.identity(fd)
.map(crate::readiness::WakeKey::Descriptor)
});
(socks.remove(&fd)?, rec, key)
};
self.retire_descriptor(fd, removed, rec, key);
Some(())
}
fn retire_descriptor(
&self,
fd: c_int,
sock: Option<Sock>,
rec: Option<Arc<SockRec>>,
key: Option<crate::readiness::WakeKey>,
) {
let regs = self.regs.clone();
let retired = rec.filter(|rec| {
regs.shared
.sockets
.detach_replaced_fd(fd, rec, regs.shared.stamp())
});
snare_interpose::defer_descriptor_cleanup(Box::new(move || {
Self::release_sock(®s, sock);
if let Some(rec) = retired {
regs.shared.sockets.finish_replaced_fd(&rec);
}
if let Some(key) = key {
regs.shared.bump_keys(&[key]);
}
}));
}
fn release_sock(regs: &Registries, sock: Option<Sock>) {
match sock {
Some(Sock::Stream { conn, end, rec, .. }) => {
match rec.opts().linger {
Some(linger) if linger.is_zero() => conn.reset(end),
Some(linger) => {
conn.close(end);
let pipe = conn.write_pipe(end).clone();
readiness().wait_until_on(
"linger",
Some(Deadline::timeout(linger)),
&[rec.wake_key()],
|| rec.pending_time(),
|| pipe.delivered(),
);
}
None if conn.read_pipe(end).has_unread() && !conn.pair => conn.reset(end),
None => conn.close(end),
}
conn.read_pipe(end).abandon_reader();
if conn.pair {
conn.read_pipe(end).close(|_| {});
}
}
Some(Sock::Dgram {
local: Some(local), ..
}) => {
regs.udp.lock().unwrap().remove(&local);
}
Some(Sock::Listener { listener, .. }) => {
unlisten(regs, &listener);
let orphans: Vec<_> = listener.pending.lock().unwrap().drain(..).collect();
for conn in orphans {
conn.reset(End::B);
}
}
_ => {}
}
}
fn recv_deadline(&self, fd: c_int) -> Option<Deadline> {
let rec = self.rec(fd)?;
Some(Deadline::timeout(rec.opts().rcvtimeo?))
}
fn rec(&self, fd: c_int) -> Option<Arc<SockRec>> {
self.socks.lock().unwrap().get(&fd)?.rec().cloned()
}
fn fd_interests(
&self,
fds: impl IntoIterator<Item = c_int>,
keys: &mut crate::readiness::WakeKeys,
) -> bool {
let mut time_sensitive = false;
for fd in fds {
let connecting = matches!(
self.socks.lock().unwrap().get(&fd),
Some(Sock::Connecting { .. })
);
if let Some(rec) = self.rec(fd).or_else(|| self.shared().sockets.lookup_fd(fd)) {
keys.push(rec.wake_key());
time_sensitive |= connecting || rec.pending_time();
} else if let Some(id) = self.socks.lock().unwrap().identity(fd) {
keys.push(crate::readiness::WakeKey::Descriptor(id));
}
}
time_sensitive
}
fn bump_fds(&self, fds: impl IntoIterator<Item = c_int>) {
let mut keys = crate::readiness::WakeKeys::default();
self.fd_interests(fds, &mut keys);
self.shared().bump_keys(keys.as_slice());
}
#[cfg(target_os = "linux")]
fn epoll_interests(&self, epfd: c_int, keys: &mut crate::readiness::WakeKeys) -> bool {
self.fd_interests([epfd], keys);
let watched: Vec<_> = {
let socks = self.socks.lock().unwrap();
let Some(Sock::Epoll { interests }) = socks.get(&epfd) else {
return false;
};
interests.keys().copied().collect()
};
let mut time_sensitive = false;
for (fd, identity) in watched {
let rec = match identity {
EpollIdentity::Local(id) => {
let socks = self.socks.lock().unwrap();
let sock = socks.get_identity(id);
time_sensitive |= matches!(sock, Some(Sock::Connecting { .. }));
if sock.is_some() {
keys.push(crate::readiness::WakeKey::Descriptor(id));
}
sock.and_then(Sock::rec).cloned()
}
EpollIdentity::Foreign(id) => self.shared().sockets.lookup_raw_id(id),
EpollIdentity::Untracked => self.shared().sockets.lookup_fd(fd),
};
if let Some(rec) = rec {
keys.push(rec.wake_key());
time_sensitive |= rec.pending_time();
}
}
time_sensitive
}
#[cfg(target_os = "macos")]
fn kqueue_interests(&self, kq: c_int, keys: &mut crate::readiness::WakeKeys) -> bool {
self.fd_interests([kq], keys);
let state = match self.socks.lock().unwrap().get(&kq) {
Some(Sock::Kqueue { state }) => state.clone(),
_ => return false,
};
let mut fds: Vec<_> = {
let state = state.lock().unwrap();
state
.reads
.keys()
.chain(state.writes.keys())
.copied()
.collect()
};
fds.sort_unstable();
fds.dedup();
self.fd_interests(fds, keys)
}
fn shared(&self) -> &SimShared {
&self.regs.shared
}
fn open_socket(
&self,
kind: SocketKind,
flags: c_int,
sock: impl FnOnce(Arc<SockRec>) -> Sock,
) -> Option<NetResult> {
match self.reserve_fd_flags(flags) {
Ok(fd) => {
let rec = self.shared().new_socket(kind, fd);
self.socks.lock().unwrap().insert(fd, sock(rec));
ok(fd as i64)
}
Err(e) => err(e.raw_os_error().unwrap_or(libc::EMFILE)),
}
}
pub(crate) fn registries(&self) -> Arc<Registries> {
self.regs.clone()
}
fn reserve_fd(&self) -> io::Result<c_int> {
let fd = unsafe { libc::dup(self.devnull) };
if fd < 0 {
return Err(io::Error::last_os_error());
}
Ok(fd)
}
fn reserve_fd_flags(&self, flags: c_int) -> io::Result<c_int> {
let fd = self.reserve_fd()?;
#[cfg(target_os = "linux")]
if flags & libc::SOCK_CLOEXEC != 0 {
let result = snare_interpose::real(|| unsafe {
libc::fcntl(fd, libc::F_SETFD, libc::FD_CLOEXEC)
});
if result < 0 {
let error = io::Error::last_os_error();
unsafe { libc::close(fd) };
return Err(error);
}
}
#[cfg(not(target_os = "linux"))]
let _ = flags;
Ok(fd)
}
fn udp_send(
&self,
fd: c_int,
endpoint: UdpSendEndpoint,
data: &[u8],
dest: SocketAddr,
flags: c_int,
) -> Option<NetResult> {
if data.len() > if dest.is_ipv4() { 65507 } else { 65527 } {
return if cfg!(target_os = "macos") && dest.is_ipv6() {
ok(data.len() as i64)
} else {
err(libc::EMSGSIZE)
};
}
let UdpSendEndpoint {
local,
requested,
queue,
domain,
broadcast,
nonblocking,
rec,
} = endpoint;
let nonblocking = nonblocking || flags & libc::MSG_DONTWAIT != 0;
if cfg!(target_os = "macos") && data.len() > rec.state().buf.sndbuf.max(0) as usize {
return err(libc::EMSGSIZE);
}
#[cfg(target_os = "linux")]
rec.land();
if let Some(errno) = rec.take_error_as(self.shared(), Taker::Send) {
return err(errno);
}
let station = self.regs.station_at(dest.ip());
let sender = match self.shared().route_send(
&rec.view(local.or(unhashed(requested))),
dest,
Op::Send,
station,
broadcast,
) {
Ok(sender) => sender,
Err(errno) => return err(errno),
};
if let Err(errno) = crate::netif::frag::check_send(
self.shared(),
&rec,
domain == libc::AF_INET6,
&sender,
dest,
data.len(),
) {
return err(errno);
}
let local = match local {
Some(local) => local,
None => {
let ip = requested.map_or(unspecified_like(loopback_for(domain)), |r| r.ip());
match self.autobind(fd, &queue, &rec, ip) {
Ok(local) => local,
Err(errno) => return err(errno),
}
}
};
let src = sender.source(local);
let stalled = || {
let policies = &self.regs.shared.policies;
policies.send_stalled(local) || policies.send_stalled(src)
};
if stalled() {
if nonblocking {
return would_block();
}
if !readiness()
.wait_until_on("udp send", None, &[rec.wake_key()], &stalled, || !stalled())
{
return err(libc::EAGAIN);
}
}
self.shared()
.capture_udp(&sender, src, dest, Dir::Out, data, None);
#[cfg(target_os = "linux")]
let now = self.shared().tstamp_now();
#[cfg(target_os = "linux")]
crate::tstamp::udp_sent(&rec, now, None, || {
crate::tstamp::looped_frame(src, dest, false, data)
});
if self.regs.deliver(&sender, src, dest, data, true) == 0 {
self.shared()
.unreachable_port(&rec, &sender, src, dest, data, station);
}
if rec.finish_udp_send(dest, &sender) {
self.shared().bump_keys(&[rec.wake_key()]);
}
ok(data.len() as i64)
}
fn autobind(
&self,
fd: c_int,
queue: &Arc<DgramQueue>,
rec: &Arc<SockRec>,
ip: IpAddr,
) -> Result<SocketAddr, c_int> {
let mut regs = self.regs.udp.lock().unwrap();
let first = queue.next_port.load(Ordering::Relaxed) as u16;
let port = regs.alloc_port_from(ip, first);
if port == 0 {
return Err(libc::EAGAIN);
}
let sa = SocketAddr::new(ip, port);
regs.insert(sa, queue.clone());
drop(regs);
if let Some(Sock::Dgram { local, .. }) = self.socks.lock().unwrap().get_mut(&fd) {
*local = Some(sa);
}
rec.set_local(sa);
Ok(sa)
}
fn rehash(
&self,
fd: c_int,
queue: &Arc<DgramQueue>,
rec: &Arc<SockRec>,
from: SocketAddr,
to: SocketAddr,
) {
if from == to {
return;
}
let mut regs = self.regs.udp.lock().unwrap();
if regs.bound.contains_key(&to) {
return;
}
regs.remove(&from);
regs.insert(to, queue.clone());
drop(regs);
if let Some(Sock::Dgram { local, .. }) = self.socks.lock().unwrap().get_mut(&fd) {
*local = Some(to);
}
rec.set_local(to);
}
fn disconnect_dgram(&self, fd: c_int) -> Option<c_int> {
let mut socks = self.socks.lock().unwrap();
let Some(Sock::Dgram {
peer,
local,
requested,
queue,
rec,
domain,
..
}) = socks.get_mut(&fd)
else {
return None;
};
let domain = *domain;
*peer = None;
rec.set_peer(None);
let Some(bound) = *local else {
return Some(domain);
};
let (requested, queue, rec) = (*requested, queue.clone(), rec.clone());
let wildcard = unspecified_like(bound.ip());
if cfg!(target_os = "linux") && requested.is_none_or(|r| r.port() == 0) {
*local = None;
drop(socks);
queue.next_port.store(
bound.port().checked_add(1).unwrap_or(49152) as u64,
Ordering::Relaxed,
);
self.regs.udp.lock().unwrap().remove(&bound);
rec.state().local = unhashed(requested).filter(|a| !a.ip().is_unspecified());
return Some(domain);
}
drop(socks);
let ip = requested
.map(|r| r.ip())
.filter(|ip| cfg!(target_os = "linux") && !ip.is_unspecified())
.unwrap_or(wildcard);
self.rehash(fd, &queue, &rec, bound, SocketAddr::new(ip, bound.port()));
Some(domain)
}
}
impl Fabric {
fn ipv6(&self, fd: c_int) -> Option<bool> {
match self.socks.lock().unwrap().get(&fd)? {
Sock::Fresh { domain, .. }
| Sock::Connecting { domain, .. }
| Sock::Dgram { domain, .. } => Some(*domain == libc::AF_INET6),
Sock::Stream { conn, .. } if conn.pair => None,
Sock::Stream { conn, end, .. } => Some(conn.local(*end).is_ipv6()),
Sock::Listener { listener, .. } => Some(listener.addr.is_ipv6()),
_ => None,
}
}
fn ts_kind(&self, fd: c_int) -> crate::tstamp::Kind {
match self.socks.lock().unwrap().get(&fd) {
Some(Sock::Stream { .. }) => crate::tstamp::Kind::Established,
Some(Sock::Fresh { .. } | Sock::Connecting { .. } | Sock::Listener { .. }) => {
crate::tstamp::Kind::Unconnected
}
_ => crate::tstamp::Kind::Datagram,
}
}
unsafe fn unmodelled_set(
&self,
rec: Option<&Arc<SockRec>>,
level: c_int,
name: c_int,
val: *const u8,
len: u32,
) -> Option<NetResult> {
let Some(rec) = rec else {
return ok(0);
};
if harmless(level, name) {
let bytes = if val.is_null() {
Vec::new()
} else {
unsafe { std::slice::from_raw_parts(val, len as usize) }.to_vec()
};
rec.keep_ignored(level, name, &bytes);
return ok(0);
}
if self
.shared()
.unmodelled_option(rec, UnmodelledOption::Set { level, name })
{
return err(libc::ENOPROTOOPT);
}
ok(0)
}
unsafe fn unmodelled_get(
&self,
rec: Option<&Arc<SockRec>>,
level: c_int,
name: c_int,
val: *mut u8,
len: *mut u32,
) -> Option<NetResult> {
if let Some(rec) = rec {
if harmless(level, name) {
match rec.ignored_value(level, name) {
Some(bytes) if !val.is_null() && !len.is_null() => unsafe {
let n = bytes.len().min(*len as usize);
std::ptr::copy_nonoverlapping(bytes.as_ptr(), val, n);
*len = n as u32;
},
_ => unsafe { write_opt(0, val, len) },
}
return ok(0);
}
if self
.shared()
.unmodelled_option(rec, UnmodelledOption::Get { level, name })
{
return err(libc::ENOPROTOOPT);
}
}
unsafe { write_opt(0, val, len) };
ok(0)
}
#[cfg(target_os = "linux")]
unsafe fn hwtstamp_ioctl(&self, request: u64, arg: i64) -> Option<NetResult> {
const SIOCSHWTSTAMP: u64 = 0x89b0;
const SIOCGHWTSTAMP: u64 = 0x89b1;
if request != SIOCSHWTSTAMP && request != SIOCGHWTSTAMP {
return None;
}
if request == SIOCSHWTSTAMP && !self.shared().sys.privileges().net_admin {
return err(libc::EPERM);
}
let base = arg as *const u8;
if base.is_null() {
return err(libc::EFAULT);
}
let name: Vec<u8> = (0..libc::IFNAMSIZ)
.map(|i| unsafe { *base.add(i) })
.take_while(|&b| b != 0)
.collect();
let known = std::str::from_utf8(&name)
.ok()
.and_then(|name| self.shared().nic(name))
.is_some();
err(if known {
libc::EOPNOTSUPP
} else {
libc::ENODEV
})
}
}
#[cfg(unix)]
pub(crate) fn harmless(level: c_int, name: c_int) -> bool {
match level {
libc::SOL_SOCKET => {
matches!(
name,
libc::SO_KEEPALIVE | libc::SO_OOBINLINE | libc::SO_DEBUG
) || (cfg!(target_os = "macos") && name == SO_NOSIGPIPE)
}
libc::IPPROTO_TCP => {
name == libc::TCP_NODELAY
|| TCP_KEEPALIVE_TIMERS.contains(&name)
|| (cfg!(target_os = "linux") && name == TCP_QUICKACK)
}
libc::IPPROTO_IP => matches!(name, libc::IP_TOS | libc::IP_TTL | libc::IP_MULTICAST_TTL),
libc::IPPROTO_IPV6 => matches!(
name,
libc::IPV6_TCLASS | libc::IPV6_UNICAST_HOPS | libc::IPV6_MULTICAST_HOPS
),
_ => false,
}
}
const SO_NOSIGPIPE: c_int = 0x1022;
const TCP_QUICKACK: c_int = 12;
#[cfg(target_os = "linux")]
const TCP_KEEPALIVE_TIMERS: [c_int; 3] = [4, 5, 6];
#[cfg(target_os = "macos")]
const TCP_KEEPALIVE_TIMERS: [c_int; 3] = [0x10, 0x101, 0x102];
const UNKNOWN_IOCTL: c_int = if cfg!(target_os = "linux") {
libc::ENOTTY
} else {
libc::ENXIO
};
impl Fabric {
#[cfg(target_os = "linux")]
unsafe fn recv_errqueue(&self, fd: c_int, hdr: *mut libc::msghdr) -> Option<NetResult> {
let (rec, v6) = match self.socks.lock().unwrap().get(&fd)? {
Sock::Dgram { rec, domain, .. } => (rec.clone(), *domain == libc::AF_INET6),
Sock::Stream { rec, conn, end, .. } => (rec.clone(), conn.local(*end).is_ipv6()),
_ => return err(libc::EAGAIN),
};
if let Some(read) = unsafe {
crate::simhost::read_error_report(&rec, hdr, v6, self.shared().tstamp_now().real)
} {
return Some(read);
}
snare_interpose::charge_latency();
err(libc::EAGAIN)
}
#[allow(clippy::unnecessary_cast)]
unsafe fn recvmsg_stream(
&self,
fd: c_int,
hdr: *mut libc::msghdr,
flags: c_int,
) -> Option<NetResult> {
let (iov, iovlen) = unsafe { ((*hdr).msg_iov, (*hdr).msg_iovlen as usize) };
let want = if iov.is_null() {
0
} else {
(0..iovlen)
.map(|i| unsafe { (*iov.add(i)).iov_len })
.sum::<usize>()
};
let direct = iovlen == 1
&& !iov.is_null()
&& want > 0
&& want <= isize::MAX as usize
&& unsafe { !(*iov).iov_base.is_null() };
let mut buf = if direct { Vec::new() } else { vec![0u8; want] };
let out = if direct {
unsafe { std::slice::from_raw_parts_mut((*iov).iov_base.cast::<u8>(), want) }
} else {
&mut buf
};
let (n, stamp) = match self.stream_read(fd, out, flags)? {
Ok(got) => got,
Err(errno) => return err(errno),
};
let copied = if direct {
n
} else {
unsafe { scatter(hdr, &buf[..n]) }
};
let cmsgs = match (stamp, self.rec(fd)) {
(Some(at), Some(rec)) => {
crate::tstamp::rx_cmsgs(Some(self.shared()), &rec, at, Rx::Stream, None)
}
_ => Vec::new(),
};
if let Some(at) = stamp.filter(|_| !cmsgs.is_empty()) {
self.shared().reach_stamp(at);
}
unsafe {
if !(*hdr).msg_name.is_null() {
(*hdr).msg_namelen = 0;
}
let truncated = write_cmsg_list(hdr, &cmsgs);
(*hdr).msg_flags = if truncated { libc::MSG_CTRUNC } else { 0 };
}
ok(copied as i64)
}
fn stream_read(
&self,
fd: c_int,
out: &mut [u8],
flags: c_int,
) -> Option<Result<(usize, Option<Stamp>), c_int>> {
let (conn, end, nonblocking, rec) = match self.socks.lock().unwrap().get(&fd)? {
Sock::Stream {
conn,
end,
nonblocking,
rec,
} => (conn.clone(), *end, *nonblocking, rec.clone()),
_ => return None,
};
#[cfg(target_os = "macos")]
conn.progress_tcp_shutdown();
let pipe = conn.read_pipe(end);
if out.is_empty() && cfg!(target_os = "macos") {
return Some(Ok((0, None)));
}
let peek = flags & libc::MSG_PEEK != 0;
let waitall =
flags & libc::MSG_WAITALL != 0 && !nonblocking && flags & libc::MSG_DONTWAIT == 0;
let mut copied = 0;
let mut stamp = None;
let deadline = rec.opts().rcvtimeo.map(Deadline::timeout);
let taker = if flags & libc::MSG_PEEK != 0 {
Taker::Peek
} else {
Taker::Recv
};
loop {
#[cfg(target_os = "macos")]
conn.progress_tcp_shutdown();
let read = if peek {
pipe.peek(out)
} else {
pipe.read(&mut out[copied..])
};
if matches!(read, Read::Shut | Read::Eof | Read::WouldBlock)
&& let Some(errno) = rec.take_error_as(self.shared(), taker)
{
return Some(if copied > 0 {
Ok((copied, stamp))
} else {
Err(errno)
});
}
match read {
Read::Data(n) => {
copied = if peek { n } else { copied + n };
if !peek {
stamp = conn.took(end, n).or(stamp);
}
if !waitall
|| copied == out.len()
|| n == 0
|| (peek && (pipe.read_eof() || rec.peek_error().is_some()))
{
return Some(Ok((copied, stamp)));
}
if peek
&& !readiness().wait_until_on(
"tcp peek",
deadline,
&[rec.wake_key()],
|| rec.pending_time(),
|| {
rec.fionread() as usize >= out.len()
|| pipe.read_eof()
|| rec.peek_error().is_some()
},
)
{
return Some(Ok((copied, stamp)));
}
}
Read::Reset => {
return Some(match rec.take_error_as(self.shared(), taker) {
Some(errno) if copied == 0 => Err(errno),
_ => Ok((copied, stamp)),
});
}
Read::Shut | Read::Eof => return Some(Ok((copied, stamp))),
Read::WouldBlock if nonblocking || flags & libc::MSG_DONTWAIT != 0 => {
snare_interpose::charge_latency();
return Some(Err(libc::EAGAIN));
}
Read::WouldBlock => {
if !readiness().wait_until_on(
"tcp recv",
deadline,
&[rec.wake_key()],
|| rec.pending_time(),
|| {
#[cfg(target_os = "macos")]
conn.progress_tcp_shutdown();
pipe.readable_or_closed() || rec.peek_error().is_some()
},
) {
return Some(if copied > 0 {
Ok((copied, stamp))
} else {
Err(libc::EAGAIN)
});
}
}
}
}
}
fn recv_dgram(
&self,
endpoint: &DgramRecvEndpoint,
flags: c_int,
) -> Result<(Datagram, u32), c_int> {
let DgramRecvEndpoint {
queue,
nonblocking,
rec,
} = endpoint;
let nonblocking = *nonblocking || flags & libc::MSG_DONTWAIT != 0;
let taker = if flags & libc::MSG_PEEK != 0 {
Taker::Peek
} else {
Taker::Recv
};
#[cfg(target_os = "linux")]
rec.land();
let error_first = cfg!(target_os = "linux");
let accept = |_: SocketAddr| true;
let deadline = if nonblocking {
None
} else {
rec.opts().rcvtimeo.map(Deadline::timeout)
};
loop {
#[cfg(target_os = "linux")]
rec.land();
if error_first && let Some(errno) = rec.take_error_as(self.shared(), taker) {
return Err(errno);
}
let got = if matches!(taker, Taker::Peek) {
queue.peek(accept)
} else {
queue.pop(accept)
};
if let Some(got) = got {
return Ok(got);
}
if let Some(errno) = rec.take_error_as(self.shared(), taker) {
return Err(errno);
}
if nonblocking {
snare_interpose::charge_latency();
return Err(libc::EAGAIN);
}
if !readiness().wait_until_on(
"udp recv",
deadline,
&[rec.wake_key()],
|| rec.pending_time(),
|| queue.has(accept) || rec.peek_error().is_some(),
) {
return Err(libc::EAGAIN); }
}
}
unsafe fn recvfrom_dgram(
&self,
endpoint: DgramRecvEndpoint,
buf: *mut u8,
len: usize,
flags: c_int,
addr: *mut u8,
addr_len: *mut u32,
) -> Option<NetResult> {
let (dg, _) = match self.recv_dgram(&endpoint, flags) {
Ok(got) => got,
Err(errno) => return err(errno),
};
#[cfg(target_os = "linux")]
crate::tstamp::note_datagram_read(self.shared(), &endpoint.rec, dg.stamp, &dg.rx_fallback);
let n = len.min(dg.data.len());
unsafe { std::ptr::copy_nonoverlapping(dg.data.as_ptr(), buf, n) };
if !addr.is_null() && !addr_len.is_null() {
if dg.src == UNNAMED {
unsafe { write_unnamed(addr, addr_len) };
} else {
unsafe { write_addr(dg.src, addr, addr_len) };
}
}
ok(
if cfg!(target_os = "linux") && flags & libc::MSG_TRUNC != 0 {
dg.data.len()
} else {
n
} as i64,
)
}
}
#[allow(clippy::unnecessary_cast)]
pub(crate) unsafe fn scatter(hdr: *const libc::msghdr, data: &[u8]) -> usize {
let (iov, iovlen) = unsafe { ((*hdr).msg_iov, (*hdr).msg_iovlen as usize) };
let mut copied = 0;
if iov.is_null() {
return 0;
}
for i in 0..iovlen {
let v = unsafe { &*iov.add(i) };
let take = (data.len() - copied).min(v.iov_len);
if take == 0 || v.iov_base.is_null() {
continue;
}
unsafe {
std::ptr::copy_nonoverlapping(data[copied..].as_ptr(), v.iov_base.cast::<u8>(), take)
};
copied += take;
}
copied
}
#[cfg(target_os = "linux")]
pub(crate) unsafe fn write_cmsgs_at(
hdr: *mut libc::msghdr,
msgs: &[(c_int, c_int, &[u8])],
) -> bool {
let owned: Vec<crate::tstamp::Cmsg> = msgs
.iter()
.map(|&(level, ty, data)| (level, ty, data.to_vec()))
.collect();
!unsafe { write_cmsg_list(hdr, &owned) }
}
pub(crate) unsafe fn write_cmsg_list(hdr: *mut libc::msghdr, msgs: &[crate::tstamp::Cmsg]) -> bool {
let control = unsafe { (*hdr).msg_control }.cast::<u8>();
let cap = if control.is_null() {
0
} else {
#[allow(clippy::unnecessary_cast)]
unsafe {
(*hdr).msg_controllen as usize
}
};
let mut used = 0;
let mut truncated = false;
for (level, ty, data) in msgs {
let space = unsafe { libc::CMSG_SPACE(data.len() as u32) } as usize;
let len = unsafe { libc::CMSG_LEN(data.len() as u32) } as usize;
let room = cap - used;
if space <= room {
unsafe {
let cmsg = control.add(used).cast::<libc::cmsghdr>();
std::ptr::write_bytes(cmsg.cast::<u8>(), 0, space);
(*cmsg).cmsg_level = *level;
(*cmsg).cmsg_type = *ty;
(*cmsg).cmsg_len = len as _;
std::ptr::copy_nonoverlapping(data.as_ptr(), libc::CMSG_DATA(cmsg), data.len());
}
used += space;
continue;
}
if cfg!(target_os = "linux") {
truncated = true;
if room >= size_of::<libc::cmsghdr>() {
let mut whole = vec![0u8; space];
unsafe {
let cmsg = whole.as_mut_ptr().cast::<libc::cmsghdr>();
(*cmsg).cmsg_level = *level;
(*cmsg).cmsg_type = *ty;
(*cmsg).cmsg_len = room as _;
std::ptr::copy_nonoverlapping(data.as_ptr(), libc::CMSG_DATA(cmsg), data.len());
std::ptr::copy_nonoverlapping(whole.as_ptr(), control.add(used), room);
}
used = cap;
}
break;
}
if room == 0 {
continue;
}
let mut whole = vec![0u8; space];
unsafe {
let cmsg = whole.as_mut_ptr().cast::<libc::cmsghdr>();
(*cmsg).cmsg_level = *level;
(*cmsg).cmsg_type = *ty;
(*cmsg).cmsg_len = len as _;
std::ptr::copy_nonoverlapping(data.as_ptr(), libc::CMSG_DATA(cmsg), data.len());
std::ptr::copy_nonoverlapping(whole.as_ptr(), control.add(used), room);
}
used = cap;
truncated = true;
}
if !control.is_null() {
unsafe { (*hdr).msg_controllen = used as _ };
}
truncated
}
pub(crate) fn duplicate_to(fd: c_int, newfd: c_int, flags: Option<c_int>) -> c_int {
snare_interpose::real(|| unsafe {
#[cfg(target_os = "linux")]
if let Some(flags) = flags {
return libc::dup3(fd, newfd, flags);
}
#[cfg(not(target_os = "linux"))]
let _ = flags;
libc::dup2(fd, newfd)
})
}
fn sock_nonblock() -> c_int {
#[cfg(target_os = "linux")]
{
libc::SOCK_NONBLOCK
}
#[cfg(not(target_os = "linux"))]
{
0
}
}
const FIONREAD: libc::c_ulong = libc::FIONREAD as libc::c_ulong;
fn is_siocoutq(request: libc::c_ulong) -> bool {
cfg!(target_os = "linux") && request == libc::TIOCOUTQ as libc::c_ulong
}
fn ok(n: i64) -> Option<NetResult> {
Some(NetResult::Ok(n))
}
fn would_block() -> Option<NetResult> {
snare_interpose::charge_latency();
err(libc::EAGAIN)
}
fn err(errno: c_int) -> Option<NetResult> {
Some(NetResult::Err(errno))
}
mod ifreq {
#[cfg(target_os = "linux")]
pub(super) const INDEX: libc::c_ulong = libc::SIOCGIFINDEX;
#[cfg(target_os = "linux")]
pub(super) const HWADDR: libc::c_ulong = libc::SIOCGIFHWADDR;
#[cfg(target_os = "linux")]
pub(super) const MTU: libc::c_ulong = libc::SIOCGIFMTU;
#[cfg(target_os = "linux")]
pub(super) const FLAGS: libc::c_ulong = libc::SIOCGIFFLAGS;
#[cfg(target_os = "macos")]
pub(super) const MTU: libc::c_ulong = 0xc020_6933;
#[cfg(target_os = "macos")]
pub(super) const FLAGS: libc::c_ulong = 0xc020_6911;
pub(super) const UNKNOWN: i32 = if cfg!(target_os = "linux") {
libc::ENODEV
} else {
libc::ENXIO
};
}
const NAMETOINDEX_UNKNOWN: i32 = if cfg!(target_os = "linux") {
libc::ENODEV
} else {
libc::ENXIO
};
impl Fabric {
unsafe fn nic_ioctl(&self, request: u64, arg: i64) -> Option<NetResult> {
let request = request as libc::c_ulong;
#[cfg(target_os = "linux")]
let modelled = [ifreq::INDEX, ifreq::MTU, ifreq::FLAGS, ifreq::HWADDR];
#[cfg(target_os = "macos")]
let modelled = [ifreq::MTU, ifreq::FLAGS];
if !modelled.contains(&request) {
return None;
}
let base = arg as *mut u8;
if base.is_null() {
return err(libc::EFAULT);
}
let mut name = Vec::new();
for i in 0..libc::IFNAMSIZ {
let b = unsafe { *base.add(i) };
if b == 0 {
break;
}
name.push(b);
}
let Some(nic) = std::str::from_utf8(&name)
.ok()
.and_then(|name| self.shared().nic(name))
else {
return err(ifreq::UNKNOWN);
};
let field = unsafe { base.add(16) };
match request {
ifreq::MTU => unsafe { field.cast::<c_int>().write_unaligned(nic.spec.mtu as c_int) },
ifreq::FLAGS => unsafe {
field
.cast::<libc::c_short>()
.write_unaligned(crate::ifaddrs::flags(&nic) as u16 as libc::c_short)
},
#[cfg(target_os = "linux")]
ifreq::INDEX => unsafe { field.cast::<c_int>().write_unaligned(nic.index as c_int) },
#[cfg(target_os = "linux")]
ifreq::HWADDR => {
const ARPHRD_ETHER: u16 = 1;
const ARPHRD_LOOPBACK: u16 = 772;
let family = if nic.loopback {
ARPHRD_LOOPBACK
} else {
ARPHRD_ETHER
};
let mut sa = [0u8; 16];
sa[..2].copy_from_slice(&family.to_ne_bytes());
sa[2..8].copy_from_slice(&nic.hw_addr());
unsafe { std::ptr::copy_nonoverlapping(sa.as_ptr(), field, sa.len()) };
}
_ => return None,
}
ok(0)
}
}
impl Net for Fabric {
fn owns(&self, fd: c_int) -> bool {
self.socks.lock().unwrap().contains_key(&fd)
}
unsafe fn socket(&self, domain: c_int, ty: c_int, protocol: c_int) -> Option<NetResult> {
let base = ty & 0xff;
let nonblocking = ty & sock_nonblock() != 0;
#[cfg(target_os = "linux")]
if domain == libc::AF_PACKET {
if base != libc::SOCK_RAW && base != libc::SOCK_DGRAM {
return None;
}
if !self.shared().sys.has_cap(crate::limits::CAP_NET_RAW) {
return err(libc::EPERM);
}
return self.open_socket(SocketKind::Packet, ty, |rec| {
rec.set_family(domain, 0);
Sock::Raw {
ifindex: None,
rx: VecDeque::new(),
arrived: 0,
nonblocking,
rec,
}
});
}
if domain != libc::AF_INET && domain != libc::AF_INET6 {
return None;
}
#[cfg(target_os = "linux")]
let resolved = match (protocol, base) {
(0, libc::SOCK_DGRAM) => libc::IPPROTO_UDP,
(0, _) => libc::IPPROTO_TCP,
(given, _) => given,
};
#[cfg(not(target_os = "linux"))]
let _ = protocol;
if base == libc::SOCK_DGRAM {
return self.open_socket(SocketKind::Udp, ty, |rec| {
#[cfg(target_os = "linux")]
rec.set_family(domain, resolved);
Sock::Dgram {
queue: DgramQueue::for_socket(&rec),
domain,
local: None,
requested: None,
peer: None,
nonblocking,
broadcast: false,
rec,
}
});
}
if base != libc::SOCK_STREAM {
return None;
}
self.open_socket(SocketKind::TcpStream, ty, |rec| {
#[cfg(target_os = "linux")]
rec.set_family(domain, resolved);
Sock::Fresh {
domain,
local: None,
nonblocking,
rec,
}
})
}
unsafe fn socketpair(
&self,
domain: c_int,
ty: c_int,
protocol: c_int,
fds: *mut c_int,
) -> Option<NetResult> {
if domain != libc::AF_UNIX {
return None;
}
let base = ty & 0xff;
if base != libc::SOCK_STREAM && base != libc::SOCK_DGRAM {
return None;
}
if protocol != 0 && (cfg!(not(target_os = "linux")) || protocol != libc::PF_UNIX) {
return err(libc::EPROTONOSUPPORT);
}
let nonblocking = ty & sock_nonblock() != 0;
let (a, b) = match (self.reserve_fd_flags(ty), self.reserve_fd_flags(ty)) {
(Ok(a), Ok(b)) => (a, b),
(a, b) => {
let mut errno = libc::EMFILE;
for r in [a, b] {
match r {
Ok(fd) => unsafe {
libc::close(fd);
},
Err(e) => errno = e.raw_os_error().unwrap_or(errno),
}
}
return err(errno);
}
};
let (ra, rb) = (
self.shared().new_socket(SocketKind::Unix, a),
self.shared().new_socket(SocketKind::Unix, b),
);
#[cfg(target_os = "linux")]
for rec in [&ra, &rb] {
rec.set_family(libc::AF_UNIX, 0);
}
let (sa, sb) = if base == libc::SOCK_STREAM {
let conn = Arc::new(Conn {
a_to_b: Pipe::new(None, None, self.shared().domain_key()),
b_to_a: Pipe::new(None, None, self.shared().domain_key()),
client: UNNAMED,
server: UNNAMED,
regs: self.regs.clone(),
hop: None,
pair: true,
tap: None,
stamps: Default::default(),
write_delays: Default::default(),
#[cfg(target_os = "macos")]
mac_shutdown: Default::default(),
});
conn.b_to_a.read_by(&ra);
conn.a_to_b.read_by(&rb);
conn.a_to_b.write_by(&ra);
conn.b_to_a.write_by(&rb);
let end = |end, rec| Sock::Stream {
conn: conn.clone(),
end,
nonblocking,
rec,
};
(end(End::A, ra), end(End::B, rb))
} else {
let (qa, qb) = (DgramQueue::for_socket(&ra), DgramQueue::for_socket(&rb));
(
Sock::UnixDgram {
peer: Arc::downgrade(&qb),
queue: qa.clone(),
nonblocking,
rec: ra,
},
Sock::UnixDgram {
peer: Arc::downgrade(&qa),
queue: qb,
nonblocking,
rec: rb,
},
)
};
let mut socks = self.socks.lock().unwrap();
socks.insert(a, sa);
socks.insert(b, sb);
drop(socks);
unsafe {
fds.write(a);
fds.add(1).write(b);
}
ok(0)
}
unsafe fn bind(&self, fd: c_int, addr: *const u8, len: u32) -> Option<NetResult> {
#[cfg(target_os = "linux")]
{
let mut socks = self.socks.lock().unwrap();
if let Some(Sock::Raw { ifindex, .. }) = socks.get_mut(&fd) {
if (len as usize) < size_of::<libc::sockaddr_ll>() {
return err(libc::EINVAL);
}
let sll = unsafe { std::ptr::read_unaligned(addr.cast::<libc::sockaddr_ll>()) };
*ifindex = Some(sll.sll_ifindex as u32);
return ok(0);
}
}
let mut socks = self.socks.lock().unwrap();
match socks.get(&fd) {
Some(Sock::Fresh {
local: None, rec, ..
}) => {
let rec = rec.clone();
let Some(want) = (unsafe { parse_addr(addr, len) }) else {
return err(libc::EINVAL);
};
if let Err(errno) = self.shared().claim_address(want.ip()) {
return err(errno);
}
if let Some(errno) = self.shared().sys.bind_denied(want) {
return err(errno);
}
let Some(want) = self.tcp_bind_addr(&socks, want, rec.opts().reuseaddr) else {
return err(libc::EADDRINUSE);
};
if let Some(Sock::Fresh { local, .. }) = socks.get_mut(&fd) {
*local = Some(want);
}
rec.set_local(want);
return ok(0);
}
Some(Sock::Fresh { .. } | Sock::Stream { .. } | Sock::Listener { .. }) => {
return err(libc::EINVAL); }
_ => {}
}
let Some(Sock::Dgram {
queue, local, rec, ..
}) = socks.get_mut(&fd)
else {
return None;
};
if local.is_some() {
return err(libc::EINVAL); }
let Some(mut want) = (unsafe { parse_addr(addr, len) }) else {
return err(libc::EINVAL);
};
if let Err(errno) = self.shared().claim_address(want.ip()) {
return err(errno);
}
if let Some(errno) = self.shared().sys.bind_denied(want) {
return err(errno);
}
let asked = want;
let (queue, rec) = (queue.clone(), rec.clone());
let mut regs = self.regs.udp.lock().unwrap();
if want.port() == 0 {
let port = regs.alloc_port(want.ip());
if port == 0 {
return err(if cfg!(target_os = "linux") {
libc::EADDRINUSE
} else {
libc::EADDRNOTAVAIL
});
}
want.set_port(port);
} else if regs.bound.contains_key(&want) {
return err(libc::EADDRINUSE);
}
regs.insert(want, queue);
drop(regs);
if let Some(Sock::Dgram {
local, requested, ..
}) = socks.get_mut(&fd)
{
*local = Some(want);
*requested = Some(asked);
}
rec.set_local(want);
ok(0)
}
unsafe fn listen(&self, fd: c_int, backlog: c_int) -> Option<NetResult> {
let maximum = self.shared().sys.limits().listen_backlog_max;
let capacity = if cfg!(target_os = "linux") {
(if backlog < 0 {
maximum
} else {
(backlog as usize).min(maximum)
})
.saturating_add(1)
} else if backlog <= 0 {
maximum
} else {
(backlog as usize).min(maximum)
};
let mut socks = self.socks.lock().unwrap();
let (domain, local, nonblocking, rec) = match socks.get(&fd)? {
Sock::Fresh {
domain,
local,
nonblocking,
rec,
} => (*domain, *local, *nonblocking, rec.clone()),
Sock::Listener { listener, .. } => {
listener.capacity.store(capacity, Ordering::Release);
return ok(0);
}
Sock::Stream { .. } => return err(libc::EINVAL),
_ => return err(libc::EOPNOTSUPP),
};
let addr = match local {
Some(addr) => addr,
None => {
let wildcard = SocketAddr::new(unspecified_like(loopback_for(domain)), 0);
match self.tcp_bind_addr(&socks, wildcard, rec.opts().reuseaddr) {
Some(addr) => addr,
None => return err(libc::EADDRINUSE),
}
}
};
let listener = Arc::new(Listener {
rec: Some(Arc::downgrade(&rec)),
addr,
pending: Mutex::default(),
capacity: AtomicUsize::new(capacity),
occupied: AtomicUsize::new(0),
tester: false,
arrivals: AtomicU64::new(0),
ready_order: AtomicU64::new(0),
ready_at: AtomicU64::new(0),
});
self.regs
.listeners
.lock()
.unwrap()
.insert(addr, listener.clone());
rec.set_kind(SocketKind::TcpListener);
rec.set_local(addr);
socks.insert(
fd,
Sock::Listener {
listener,
nonblocking,
rec,
},
);
ok(0)
}
unsafe fn accept(
&self,
fd: c_int,
addr: *mut u8,
addr_len: *mut u32,
flags: c_int,
) -> Option<NetResult> {
let (listener, nonblocking, lrec) = match self.socks.lock().unwrap().get(&fd)? {
Sock::Listener {
listener,
nonblocking,
rec,
} => (listener.clone(), *nonblocking, rec.clone()),
Sock::Fresh { .. } | Sock::Stream { .. } => return err(libc::EINVAL),
_ => return err(libc::EOPNOTSUPP),
};
let pop = || listener.pending.lock().unwrap().pop_front();
let conn = if let Some(conn) = pop() {
conn
} else if nonblocking {
return would_block();
} else {
let deadline = if cfg!(target_os = "linux") {
lrec.opts().rcvtimeo.map(Deadline::timeout)
} else {
None
};
if !readiness().wait_until_on(
"accept",
deadline,
&[lrec.wake_key()],
|| lrec.pending_time(),
|| listener.has_pending(),
) {
return err(libc::EAGAIN);
}
match pop() {
Some(conn) => conn,
None => return err(libc::EAGAIN),
}
};
snare_interpose::descriptor_transaction(|| {
let new_fd = match self.reserve_fd_flags(flags) {
Ok(new_fd) => new_fd,
Err(e) => {
listener.pending.lock().unwrap().push_front(conn);
return err(e.raw_os_error().unwrap_or(libc::EMFILE));
}
};
listener.occupied.fetch_sub(1, Ordering::AcqRel);
let rec = self.shared().new_socket(SocketKind::TcpStream, new_fd);
rec.set_ends(conn.server, conn.client);
rec.set_listener(lrec.id);
rec.inherit_buffers(&lrec);
#[cfg(target_os = "linux")]
{
let (family, busy_poll) = {
let listener = lrec.state();
(listener.family, listener.opts.busy_poll)
};
rec.set_family(family.0, family.1);
rec.state().opts.busy_poll = busy_poll;
}
rec.note_conn_nic(self.shared().hop_name(conn.hop.as_ref()).as_deref());
conn.a_to_b.read_by(&rec);
conn.b_to_a.write_by(&rec);
#[cfg(target_os = "macos")]
conn.attach_tcp_probe(End::B, &rec);
let peer = conn.client;
self.socks.lock().unwrap().insert(
new_fd,
Sock::Stream {
conn,
end: End::B,
nonblocking: flags & sock_nonblock() != 0,
rec,
},
);
if !addr.is_null() && !addr_len.is_null() {
unsafe { write_addr(peer, addr, addr_len) };
}
ok(new_fd as i64)
})
}
unsafe fn connect(&self, fd: c_int, addr: *const u8, len: u32) -> Option<NetResult> {
if !Net::owns(self, fd) {
return None;
}
if !addr.is_null() && len as usize >= size_of::<libc::sa_family_t>() {
let family = unsafe { (*addr.cast::<libc::sockaddr>()).sa_family } as c_int;
if family == libc::AF_UNSPEC
&& let Some(domain) = self.disconnect_dgram(fd)
{
if cfg!(not(target_os = "macos")) {
return ok(0);
}
let fits = len as usize
== if domain == libc::AF_INET6 {
size_of::<libc::sockaddr_in6>()
} else {
size_of::<libc::sockaddr_in>()
};
return err(if fits {
libc::EAFNOSUPPORT
} else {
libc::EINVAL
});
}
}
let Some(server) = (unsafe { parse_addr(addr, len) }) else {
return err(libc::EINVAL);
};
let station = self.regs.station_at(server.ip());
let (domain, nonblocking, bound, rec) = {
let socks = self.socks.lock().unwrap();
match socks.get(&fd)? {
Sock::Dgram { peer: Some(_), .. } if cfg!(target_os = "macos") => {
drop(socks);
self.disconnect_dgram(fd);
return unsafe { self.connect(fd, addr, len) };
}
Sock::Dgram {
local,
requested,
queue,
rec,
broadcast,
..
} => {
let (local, requested, queue, rec, broadcast) =
(*local, *requested, queue.clone(), rec.clone(), *broadcast);
drop(socks);
let sender = match self.shared().route_send(
&rec.view(local.or(unhashed(requested))),
server,
Op::Connect,
station,
broadcast,
) {
Ok(sender) => sender,
Err(errno) => return err(errno),
};
match local {
None => {
let unbound = unhashed(requested)
.unwrap_or(SocketAddr::new(unspecified_like(server.ip()), 0));
let ip = sender.connected_local(unbound).ip();
if let Err(errno) = self.autobind(fd, &queue, &rec, ip) {
return err(errno);
}
}
Some(bound) => {
self.rehash(fd, &queue, &rec, bound, sender.connected_local(bound))
}
}
if let Some(Sock::Dgram { peer, .. }) = self.socks.lock().unwrap().get_mut(&fd)
{
*peer = Some(server);
}
rec.set_peer(Some(server));
rec.note_tx_nic(sender.egress_name());
return ok(0);
}
Sock::Fresh {
domain,
nonblocking,
local,
rec,
} => (*domain, *nonblocking, *local, rec.clone()),
#[cfg(target_os = "linux")]
Sock::Connecting {
nonblocking: false, ..
} => {
drop(socks);
return self.await_connect(fd);
}
Sock::Connecting { .. } => {
drop(socks);
return match self.advance_connect(fd) {
Progress::Pending(_) => {
snare_interpose::charge_latency();
err(libc::EALREADY)
}
Progress::Settled(Outcome::Failed(errno), rec) => {
rec.take_error();
err(errno)
}
Progress::Settled(Outcome::Connected, _) => err(libc::EISCONN),
Progress::Idle => unsafe { self.connect(fd, addr, len) },
};
}
Sock::Stream { rec, .. }
if cfg!(target_os = "linux") && rec.state().connect_ack =>
{
rec.state().connect_ack = false;
return ok(0);
}
Sock::Stream { .. } | Sock::Listener { .. } => return err(libc::EISCONN),
_ => return err(libc::EOPNOTSUPP),
}
};
rec.state().tcp_failed = false;
let sender =
match self
.shared()
.route_send(&rec.view(bound), server, Op::Connect, station, true)
{
Ok(sender) => sender,
Err(errno) => return err(errno),
};
if bound == Some(server) {
if cfg!(target_os = "macos") {
return err(libc::EINVAL);
}
let pipe = Pipe::new(None, None, self.shared().domain_key());
pipe.read_by(&rec);
pipe.write_by(&rec);
rec.set_ends(server, server);
let conn = Arc::new(Conn {
a_to_b: pipe.clone(),
b_to_a: pipe,
client: server,
server,
regs: self.regs.clone(),
hop: None,
pair: false,
tap: None,
stamps: Default::default(),
write_delays: Default::default(),
#[cfg(target_os = "macos")]
mac_shutdown: Default::default(),
});
self.socks.lock().unwrap().insert(
fd,
Sock::Stream {
conn,
end: End::A,
nonblocking,
rec,
},
);
return if nonblocking {
err(libc::EINPROGRESS)
} else {
ok(0)
};
}
crate::netstats::bump(&self.shared().stats.tcp(server.ip()).active_opens);
let host_local = matches!(&sender, Sender::Host(path) if path.local);
let listener = self.listener_for(server, host_local);
let mut syn = self.shared().syn_probe(server, listener.is_some(), station);
let admission = if matches!(syn, Syn::Accept) {
listener.as_ref().and_then(Listener::reserve)
} else {
None
};
if matches!(syn, Syn::Accept) && admission.is_none() {
syn = Syn::Silent;
}
let client = self.client_addr(server, bound, &sender);
if client.port() == 0 {
return err(libc::EADDRNOTAVAIL);
}
if let (Syn::Accept, Some(listener)) = (syn, &listener) {
let conn = self.establish(&rec, server, client, &sender, listener.tester);
if let Some(lrec) = listener.rec.as_ref().and_then(std::sync::Weak::upgrade) {
conn.a_to_b.pending_reader(lrec.state().buf);
}
if let Some(tap) = &conn.tap {
tap.open(false, Duration::ZERO);
}
self.socks.lock().unwrap().insert(
fd,
Sock::Stream {
conn: conn.clone(),
end: End::A,
nonblocking,
rec,
},
);
admission
.expect("accepted SYN reserves a queue entry")
.queue(conn);
let mut keys = crate::readiness::WakeKeys::default();
self.fd_interests([fd], &mut keys);
if let Some(key) = listener.wake_key() {
keys.push(key);
}
self.shared().bump_keys(keys.as_slice());
if nonblocking {
if let Some(rec) = self.rec(fd) {
rec.state().connect_ack = cfg!(target_os = "linux");
}
return err(libc::EINPROGRESS);
}
return ok(0);
}
let plan = self.shared().syn_plan(&rec.opts(), syn);
let server_in = listener.as_ref().is_none_or(|l| l.tester);
let tap = self.shared().tcp_tap(&sender, client, server, server_in);
let attempt = match ConnectAttempt::start(&plan, syn, client, tap) {
Ok(attempt) => attempt,
Err(Outcome::Failed(errno)) => {
self.shared()
.connect_settled(server, Outcome::Failed(errno), false);
if errno == libc::ECONNREFUSED && host_local {
crate::netstats::bump(&self.shared().stats.tcp(server.ip()).out_rsts);
}
if nonblocking {
rec.state().tcp_failed = true;
rec.set_pending_error(errno, None);
return err(libc::EINPROGRESS);
}
return err(errno);
}
Err(Outcome::Connected) => unreachable!("a SYN a listener accepts connects at once"),
};
self.socks.lock().unwrap().insert(
fd,
Sock::Connecting {
domain,
local: bound,
nonblocking,
rec,
dest: server,
attempt: Arc::new(Mutex::new(attempt)),
},
);
if nonblocking {
return err(libc::EINPROGRESS);
}
self.await_connect(fd)
}
unsafe fn send(
&self,
fd: c_int,
buf: *const u8,
len: usize,
flags: c_int,
) -> Option<NetResult> {
let socks = self.socks.lock().unwrap();
match socks.get(&fd) {
Some(Sock::Stream {
conn,
end,
rec,
nonblocking,
}) => {
let bytes = unsafe { std::slice::from_raw_parts(buf, len) };
let (conn, end, rec) = (conn.clone(), *end, rec.clone());
let nonblocking = *nonblocking || flags & libc::MSG_DONTWAIT != 0;
drop(socks);
#[cfg(target_os = "macos")]
conn.progress_tcp_shutdown();
if (cfg!(target_os = "macos") && conn.write_pipe(end).is_write_shut())
|| (conn.is_reset() && conn.write_pipe(end).reader_gone())
{
return err(libc::EPIPE);
}
if let Some(errno) = rec.take_error_as(self.shared(), Taker::Send) {
return err(errno);
}
let sent = conn.write_pipe(end).send(
bytes,
nonblocking,
rec.opts().sndtimeo,
|rest| conn.write(end, rest),
|| {
#[cfg(target_os = "macos")]
conn.progress_tcp_shutdown();
rec.peek_error().is_some()
},
);
match sent {
Sent::Bytes(n) => {
rec.count_sent();
#[cfg(target_os = "linux")]
if !conn.pair {
conn.write_pipe(end).timestamp_write(
self.shared().tstamp_now(),
&bytes[..n],
Duration::from_nanos(
conn.write_delays[end.index()].load(Ordering::Relaxed),
) * 2,
);
}
ok(n as i64)
}
Sent::Closed if conn.is_reset() => err(rec
.take_error_as(self.shared(), Taker::Send)
.unwrap_or(libc::EPIPE)),
Sent::Closed => err(libc::EPIPE),
Sent::WouldBlock | Sent::TimedOut | Sent::Stuck => err(libc::EAGAIN),
Sent::Error => err(rec
.take_error_as(self.shared(), Taker::Send)
.unwrap_or(libc::EAGAIN)),
}
}
Some(Sock::UnixDgram { peer, rec, .. }) => {
let (peer, rec) = (peer.upgrade(), rec.clone());
drop(socks);
let Some(peer) = peer else {
return err(PAIR_PEER_GONE);
};
let data = unsafe { std::slice::from_raw_parts(buf, len) };
peer.push_after(
UNNAMED,
data,
Duration::ZERO,
None,
self.shared().tstamp_now(),
None,
self.shared().domain_key(),
);
rec.count_sent();
ok(len as i64)
}
Some(sock @ Sock::Dgram { peer, .. }) => {
let Some(dest) = *peer else {
return err(libc::EDESTADDRREQ);
};
let endpoint = UdpSendEndpoint::from_sock(sock).unwrap();
drop(socks);
let data = unsafe { std::slice::from_raw_parts(buf, len) };
self.udp_send(fd, endpoint, data, dest, flags)
}
Some(_) => err(libc::ENOTCONN),
None => None,
}
}
unsafe fn sendto(
&self,
fd: c_int,
buf: *const u8,
len: usize,
flags: c_int,
addr: *const u8,
addr_len: u32,
) -> Option<NetResult> {
if addr.is_null() {
return unsafe { self.send(fd, buf, len, flags) };
}
let endpoint = {
let socks = self.socks.lock().unwrap();
UdpSendEndpoint::from_sock(socks.get(&fd)?)?
};
let Some(dest) = (unsafe { parse_addr(addr, addr_len) }) else {
return err(libc::EINVAL);
};
let data = unsafe { std::slice::from_raw_parts(buf, len) };
self.udp_send(fd, endpoint, data, dest, flags)
}
unsafe fn sendmsg(&self, fd: c_int, msg: *const u8, flags: c_int) -> Option<NetResult> {
if !Net::owns(self, fd) {
return None;
}
if msg.is_null() {
return err(libc::EFAULT);
}
let hdr = msg as *const libc::msghdr;
let (name, name_len, iov, iov_len, control_len) = unsafe {
(
(*hdr).msg_name,
(*hdr).msg_namelen,
(*hdr).msg_iov,
(*hdr).msg_iovlen,
(*hdr).msg_controllen,
)
};
if control_len != 0 {
return err(libc::EOPNOTSUPP);
}
let mut data = Vec::new();
if !iov.is_null() {
for i in 0..iov_len as _ {
let v = unsafe { &*iov.add(i) };
if !v.iov_base.is_null() && v.iov_len != 0 {
data.extend_from_slice(unsafe {
std::slice::from_raw_parts(v.iov_base as *const u8, v.iov_len)
});
}
}
}
unsafe {
self.sendto(
fd,
data.as_ptr(),
data.len(),
flags,
name as *const u8,
name_len,
)
}
}
unsafe fn recvfrom(
&self,
fd: c_int,
buf: *mut u8,
len: usize,
flags: c_int,
addr: *mut u8,
addr_len: *mut u32,
) -> Option<NetResult> {
let endpoint = {
let socks = self.socks.lock().unwrap();
DgramRecvEndpoint::from_sock(socks.get(&fd)?)?
};
unsafe { self.recvfrom_dgram(endpoint, buf, len, flags, addr, addr_len) }
}
unsafe fn recvmsg(&self, fd: c_int, msg: *mut u8, flags: c_int) -> Option<NetResult> {
if msg.is_null() {
return Net::owns(self, fd).then_some(NetResult::Err(libc::EFAULT));
}
#[cfg(target_os = "linux")]
if flags & libc::MSG_ERRQUEUE != 0 {
return unsafe { self.recv_errqueue(fd, msg.cast()) };
}
let endpoint = {
let socks = self.socks.lock().unwrap();
let sock = socks.get(&fd)?;
if matches!(sock, Sock::Stream { .. }) {
None
} else {
Some(DgramRecvEndpoint::from_sock(sock)?)
}
};
let Some(endpoint) = endpoint else {
return unsafe { self.recvmsg_stream(fd, msg.cast(), flags) };
};
#[cfg(target_os = "linux")]
let rxq_ovfl = endpoint.rec.state().buf.rxq_ovfl;
let (dg, drops) = match self.recv_dgram(&endpoint, flags) {
Ok(got) => got,
Err(errno) => return err(errno),
};
let hdr = msg.cast::<libc::msghdr>();
let copied = unsafe { scatter(hdr, &dg.data) };
let mut msg_flags = 0;
if copied < dg.data.len() {
msg_flags |= libc::MSG_TRUNC;
}
unsafe {
let name = (*hdr).msg_name.cast::<u8>();
if !name.is_null() {
let mut name_len = (*hdr).msg_namelen;
if dg.src == UNNAMED {
write_unnamed(name, &mut name_len);
} else {
write_addr(dg.src, name, &mut name_len);
}
(*hdr).msg_namelen = name_len;
}
}
#[cfg(target_os = "linux")]
let fallback = Some(dg.rx_fallback.as_ref());
#[cfg(not(target_os = "linux"))]
let fallback = None;
#[allow(unused_mut)]
let mut cmsgs = crate::tstamp::rx_cmsgs(
Some(self.shared()),
&endpoint.rec,
dg.stamp,
Rx::Datagram,
fallback,
);
if !cmsgs.is_empty() {
self.shared().reach_stamp(dg.stamp);
}
#[cfg(target_os = "linux")]
if rxq_ovfl && drops > 0 {
cmsgs.push((
libc::SOL_SOCKET,
crate::limits::SO_RXQ_OVFL,
drops.to_ne_bytes().to_vec(),
));
}
#[cfg(not(target_os = "linux"))]
let _ = drops;
if unsafe { write_cmsg_list(hdr, &cmsgs) } {
msg_flags |= libc::MSG_CTRUNC;
}
unsafe { (*hdr).msg_flags = msg_flags };
let real_len = cfg!(target_os = "linux") && flags & libc::MSG_TRUNC != 0;
ok(if real_len { dg.data.len() } else { copied } as i64)
}
unsafe fn recv(&self, fd: c_int, buf: *mut u8, len: usize, flags: c_int) -> Option<NetResult> {
let endpoint = {
let socks = self.socks.lock().unwrap();
match socks.get(&fd)? {
Sock::Stream { .. } => None,
sock => match DgramRecvEndpoint::from_sock(sock) {
Some(endpoint) => Some(endpoint),
None => return err(libc::ENOTCONN),
},
}
};
if let Some(endpoint) = endpoint {
return unsafe {
self.recvfrom_dgram(
endpoint,
buf,
len,
flags,
std::ptr::null_mut(),
std::ptr::null_mut(),
)
};
}
let out = unsafe { std::slice::from_raw_parts_mut(buf, len) };
match self.stream_read(fd, out, flags)? {
Ok((n, _)) => ok(n as i64),
Err(errno) => err(errno),
}
}
unsafe fn shutdown(&self, fd: c_int, how: c_int) -> Option<NetResult> {
let socks = self.socks.lock().unwrap();
let sock = socks.get(&fd)?;
if ![libc::SHUT_RD, libc::SHUT_WR, libc::SHUT_RDWR].contains(&how) {
return err(libc::EINVAL);
}
match sock {
Sock::Stream { conn, end, .. } => {
let (conn, end) = (conn.clone(), *end);
drop(socks);
#[cfg(target_os = "macos")]
conn.progress_tcp_shutdown();
if how != libc::SHUT_WR {
if cfg!(target_os = "macos")
&& (conn.read_pipe(end).is_read_shut() || conn.read_pipe(end).is_closed())
{
return err(libc::ENOTCONN);
}
#[cfg(target_os = "macos")]
conn.shut_read_tcp(end);
#[cfg(not(target_os = "macos"))]
conn.read_pipe(end).shut_read();
}
if how != libc::SHUT_RD {
#[cfg(target_os = "macos")]
if conn.write_pipe(end).is_write_closed() {
return err(libc::ENOTCONN);
}
conn.close(end);
}
ok(0)
}
Sock::Fresh { .. } | Sock::Listener { .. } | Sock::Dgram { peer: None, .. } => {
err(libc::ENOTCONN)
}
_ => ok(0),
}
}
unsafe fn close(&self, fd: c_int) -> Option<NetResult> {
self.detach_fd(fd)?;
let ret = unsafe { libc::close(fd) };
ok(ret as i64)
}
unsafe fn fd_replaced(&self, fd: c_int) -> Option<NetResult> {
self.detach_fd(fd)?;
ok(0)
}
unsafe fn dup_to(&self, fd: c_int, newfd: c_int, flags: Option<c_int>) -> Option<NetResult> {
let mut socks = self.socks.lock().unwrap();
let rec = socks.get(&fd)?.rec().cloned();
let result = duplicate_to(fd, newfd, flags);
if result < 0 {
return err(io::Error::last_os_error()
.raw_os_error()
.unwrap_or(libc::EBADF));
}
if fd == newfd || socks.descriptors.get(&fd) == socks.descriptors.get(&newfd) {
return ok(result as i64);
}
let previous = socks.get(&newfd).and_then(Sock::rec).cloned();
let previous_key = previous.as_ref().map(|rec| rec.wake_key()).or_else(|| {
socks
.identity(newfd)
.map(crate::readiness::WakeKey::Descriptor)
});
let removed = socks.remove(&newfd).flatten();
socks.alias(fd, newfd);
if let Some(rec) = rec {
self.shared().sockets.alias(newfd, &rec);
}
drop(socks);
self.retire_descriptor(newfd, removed, previous, previous_key);
ok(result as i64)
}
unsafe fn getpeername(
&self,
fd: c_int,
addr: *mut u8,
addr_len: *mut u32,
) -> Option<NetResult> {
let socks = self.socks.lock().unwrap();
let peer = match socks.get(&fd)? {
Sock::Stream { conn, .. } if conn.pair => None,
Sock::UnixDgram { .. } => None,
Sock::Stream { conn, end, .. } => Some(conn.peer(*end)),
Sock::Dgram {
peer: Some(peer), ..
} => Some(*peer),
_ => return err(libc::ENOTCONN),
};
drop(socks);
match peer {
Some(peer) => unsafe { write_addr(peer, addr, addr_len) },
None => unsafe { write_unnamed(addr, addr_len) },
}
ok(0)
}
unsafe fn getsockname(
&self,
fd: c_int,
addr: *mut u8,
addr_len: *mut u32,
) -> Option<NetResult> {
let socks = self.socks.lock().unwrap();
let local = match socks.get(&fd)? {
Sock::Stream { conn, .. } if conn.pair => None,
Sock::UnixDgram { .. } => None,
Sock::Stream { conn, end, .. } => Some(conn.local(*end)),
Sock::Listener { listener, .. } => Some(listener.addr),
Sock::Dgram {
local: None,
requested: Some(requested),
..
} => Some(SocketAddr::new(requested.ip(), 0)),
Sock::Dgram { local, domain, .. }
| Sock::Fresh { local, domain, .. }
| Sock::Connecting { local, domain, .. } => Some(
local
.unwrap_or_else(|| SocketAddr::new(unspecified_like(loopback_for(*domain)), 0)),
),
_ => return None,
};
drop(socks);
match local {
Some(local) => unsafe { write_addr(local, addr, addr_len) },
None => unsafe { write_unnamed(addr, addr_len) },
}
ok(0)
}
unsafe fn setsockopt(
&self,
fd: c_int,
level: c_int,
name: c_int,
val: *const u8,
len: u32,
) -> Option<NetResult> {
if !Net::owns(self, fd) {
return None;
}
if let Some(rec) = self.rec(fd)
&& let Some(result) = unsafe {
crate::netif::sockopt::set(self.shared(), &rec, level, name, val, len).or_else(
|| crate::limits::sockopt::set(self.shared(), &rec, level, name, val, len),
)
}
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
if let Some(rec) = self.rec(fd)
&& let Some(v6) = self.ipv6(fd)
&& let Some(result) =
unsafe { crate::netif::frag::set(&rec, v6, level, name, val, len) }
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
if level == libc::IPPROTO_TCP
&& let Some(rec) = self.rec(fd)
&& let Some(result) = unsafe { crate::faults::sockopt::set(&rec, name, val, len) }
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
if let Some(rec) = self.rec(fd)
&& let Some(result) = unsafe {
crate::tstamp::set(
Some(self.shared()),
&rec,
self.ts_kind(fd),
level,
name,
val,
len,
)
}
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
#[cfg(target_os = "linux")]
if let Some(rec) = self.rec(fd) {
let net_admin = self.shared().sys.has_cap(crate::limits::CAP_NET_ADMIN);
if let Some(result) =
unsafe { crate::tstamp::set_txtime(&rec, net_admin, level, name, val, len) }
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
}
if level == libc::SOL_SOCKET && name == libc::SO_BROADCAST {
let on = unsafe { read_int(val, len) } != 0;
if let Some(Sock::Dgram { broadcast, .. }) = self.socks.lock().unwrap().get_mut(&fd) {
*broadcast = on;
}
return ok(0);
}
if level == libc::SOL_SOCKET
&& let Some(rec) = self.rec(fd)
{
let timeout = || unsafe { parse_timeval(val, len) }.filter(|d| !d.is_zero());
let mut state = rec.state();
match name {
libc::SO_RCVTIMEO => state.opts.rcvtimeo = timeout(),
libc::SO_SNDTIMEO => state.opts.sndtimeo = timeout(),
libc::SO_REUSEADDR => state.opts.reuseaddr = unsafe { read_int(val, len) } != 0,
libc::SO_LINGER => {
state.opts.linger = unsafe { parse_linger(val, len, LINGER_UNIT) }
}
#[cfg(target_os = "macos")]
SO_LINGER_SEC => {
state.opts.linger = unsafe { parse_linger(val, len, Duration::from_secs(1)) }
}
_ => {
drop(state);
return unsafe { self.unmodelled_set(Some(&rec), level, name, val, len) };
}
}
return ok(0);
}
#[cfg(target_os = "linux")]
if (level, name) == (libc::IPPROTO_IP, libc::IP_RECVERR)
|| (level, name) == (libc::IPPROTO_IPV6, libc::IPV6_RECVERR)
{
if let Some(rec) = self.rec(fd) {
rec.state().dgram.recverr = unsafe { read_int(val, len) } != 0;
}
return ok(0);
}
let drop_group = (level == libc::IPPROTO_IP && name == libc::IP_DROP_MEMBERSHIP)
|| (level == libc::IPPROTO_IPV6 && name == IPV6_LEAVE);
let membership_name = if drop_group {
if level == libc::IPPROTO_IP {
libc::IP_ADD_MEMBERSHIP
} else {
IPV6_JOIN
}
} else {
name
};
if let Some(membership) = unsafe { parse_add_membership(level, membership_name, val, len) }
{
if let Some(rec) = self.rec(fd) {
return match rec.change_membership(membership, !drop_group) {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
return err(libc::ENOTSOCK);
}
unsafe { self.unmodelled_set(self.rec(fd).as_ref(), level, name, val, len) }
}
unsafe fn getsockopt(
&self,
fd: c_int,
level: c_int,
name: c_int,
val: *mut u8,
len: *mut u32,
) -> Option<NetResult> {
if !Net::owns(self, fd) {
return None;
}
if level == libc::SOL_SOCKET && name == libc::SO_ERROR {
self.advance_connect(fd);
}
let rec = self.rec(fd);
#[cfg(any(target_os = "linux", target_os = "macos"))]
if let Some(rec) = &rec {
rec.land();
}
if let Some(rec) = &rec
&& unsafe { crate::tstamp::get(rec, level, name, val, len) }.is_some()
{
return ok(0);
}
if let Some(rec) = &rec
&& level == libc::SOL_SOCKET
&& let Some(unit) = linger_unit(name)
{
unsafe { write_linger(rec.opts().linger, unit, val, len) };
return ok(0);
}
if let Some(rec) = &rec
&& unsafe { crate::netif::sockopt::get(rec, level, name, val, len) }.is_some()
{
return ok(0);
}
if let Some(rec) = &rec
&& let Some(v6) = self.ipv6(fd)
&& let Some(result) = unsafe { crate::netif::frag::get(rec, v6, level, name, val, len) }
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
if level == libc::IPPROTO_TCP
&& let Some(rec) = &rec
&& let Some(value) = crate::faults::sockopt::get(self.shared(), rec, name)
{
unsafe { write_opt(value, val, len) };
return ok(0);
}
if let Some(rec) = &rec
&& let Some(result) = unsafe { crate::limits::sockopt::get(rec, level, name, val, len) }
{
return match result {
Ok(()) => ok(0),
Err(errno) => err(errno),
};
}
if level == libc::SOL_SOCKET && (name == libc::SO_RCVTIMEO || name == libc::SO_SNDTIMEO) {
let opts = rec.map(|rec| rec.opts()).unwrap_or_default();
let d = if name == libc::SO_RCVTIMEO {
opts.rcvtimeo
} else {
opts.sndtimeo
};
unsafe { write_timeval(d.unwrap_or_default(), val, len) };
return ok(0);
}
let value: i32 = if level != libc::SOL_SOCKET {
return unsafe { self.unmodelled_get(rec.as_ref(), level, name, val, len) };
} else {
match name {
libc::SO_TYPE => match self.kind(fd) {
Some(Kind::Dgram) => libc::SOCK_DGRAM,
_ => libc::SOCK_STREAM,
},
libc::SO_BROADCAST => matches!(
self.socks.lock().unwrap().get(&fd),
Some(Sock::Dgram {
broadcast: true,
..
})
) as i32,
libc::SO_ERROR => rec
.and_then(|rec| rec.take_error_as(self.shared(), Taker::SoError))
.unwrap_or(0),
libc::SO_REUSEADDR => rec.is_some_and(|rec| rec.opts().reuseaddr) as i32,
libc::SO_ACCEPTCONN => (self.kind(fd) == Some(Kind::Listener)) as i32,
_ => return unsafe { self.unmodelled_get(rec.as_ref(), level, name, val, len) },
}
};
unsafe { write_opt(value, val, len) };
ok(0)
}
unsafe fn dup(&self, fd: c_int) -> Option<NetResult> {
unsafe { Net::fcntl(self, fd, libc::F_DUPFD, 0) }
}
unsafe fn fcntl(&self, fd: c_int, cmd: c_int, arg: i64) -> Option<NetResult> {
if cmd == libc::F_DUPFD || cmd == libc::F_DUPFD_CLOEXEC {
let mut socks = self.socks.lock().unwrap();
let sock = socks.get(&fd)?;
let rec = sock.rec().cloned();
let new_fd = snare_interpose::real(|| unsafe { libc::fcntl(fd, cmd, arg as c_int) });
if new_fd < 0 {
return err(io::Error::last_os_error()
.raw_os_error()
.unwrap_or(libc::EMFILE));
}
socks.alias(fd, new_fd);
if let Some(rec) = rec {
self.shared().sockets.alias(new_fd, &rec);
}
return ok(new_fd as i64);
}
if cmd == libc::F_GETFD || cmd == libc::F_SETFD {
if !self.socks.lock().unwrap().contains_key(&fd) {
return None;
}
let result = snare_interpose::real(|| unsafe { libc::fcntl(fd, cmd, arg as c_int) });
return if result < 0 {
err(io::Error::last_os_error()
.raw_os_error()
.unwrap_or(libc::EBADF))
} else {
ok(result as i64)
};
}
let mut socks = self.socks.lock().unwrap();
if matches!(socks.get(&fd)?, Sock::Epoll { .. } | Sock::Kqueue { .. }) {
let status = &mut socks.description_mut(&fd).unwrap().status_flags;
return match cmd {
libc::F_GETFL => ok(*status as i64),
libc::F_SETFL => {
*status = libc::O_RDWR | (arg as c_int & libc::O_NONBLOCK);
ok(0)
}
_ => err(libc::EINVAL),
};
}
let nb = match socks.get_mut(&fd) {
Some(Sock::Fresh { nonblocking, .. })
| Some(Sock::Connecting { nonblocking, .. })
| Some(Sock::Stream { nonblocking, .. })
| Some(Sock::Listener { nonblocking, .. })
| Some(Sock::Dgram { nonblocking, .. })
| Some(Sock::UnixDgram { nonblocking, .. })
| Some(Sock::Event { nonblocking, .. })
| Some(Sock::Raw { nonblocking, .. }) => nonblocking,
_ => return None,
};
match cmd {
libc::F_GETFL => {
let flags = libc::O_RDWR | if *nb { libc::O_NONBLOCK } else { 0 };
ok(flags as i64)
}
libc::F_SETFL => {
*nb = arg as c_int & libc::O_NONBLOCK != 0;
ok(0)
}
_ => err(libc::EINVAL),
}
}
unsafe fn ioctl(&self, fd: c_int, request: u64, arg: i64) -> Option<NetResult> {
if let Some(r) = unsafe { self.nic_ioctl(request, arg) } {
return Some(r);
}
if request as libc::c_ulong == FIONREAD || is_siocoutq(request as libc::c_ulong) {
let rec = match self.socks.lock().unwrap().get(&fd) {
Some(
sock @ (Sock::Stream { .. } | Sock::Dgram { .. } | Sock::UnixDgram { .. }),
) => sock.rec().cloned(),
Some(_) => None,
None => return None,
};
if arg == 0 {
return err(libc::EFAULT);
}
let value = match rec {
Some(rec) if request as libc::c_ulong == FIONREAD => rec.fionread(),
_ => 0,
};
unsafe { (arg as *mut c_int).write_unaligned(value) };
return ok(0);
}
let mut socks = self.socks.lock().unwrap();
let nb = match socks.get_mut(&fd) {
Some(Sock::Fresh { nonblocking, .. })
| Some(Sock::Connecting { nonblocking, .. })
| Some(Sock::Stream { nonblocking, .. })
| Some(Sock::Listener { nonblocking, .. })
| Some(Sock::Dgram { nonblocking, .. })
| Some(Sock::UnixDgram { nonblocking, .. })
| Some(Sock::Raw { nonblocking, .. }) => nonblocking,
_ => return None,
};
if request as libc::c_ulong == libc::FIONBIO {
let on = unsafe { *(arg as *const c_int) };
*nb = on != 0;
return ok(0);
}
let rec = socks.get(&fd).and_then(Sock::rec).cloned();
drop(socks);
if [libc::FIOCLEX, libc::FIONCLEX].contains(&(request as libc::c_ulong)) {
let result = snare_interpose::real(|| unsafe {
libc::fcntl(
fd,
libc::F_SETFD,
if request as libc::c_ulong == libc::FIOCLEX {
libc::FD_CLOEXEC
} else {
0
},
)
});
return if result < 0 {
err(io::Error::last_os_error()
.raw_os_error()
.unwrap_or(libc::EBADF))
} else {
ok(result as i64)
};
}
#[cfg(target_os = "linux")]
if let Some(r) = unsafe { self.hwtstamp_ioctl(request, arg) } {
return Some(r);
}
match rec {
Some(rec)
if self
.shared()
.unmodelled_option(&rec, UnmodelledOption::Ioctl { request }) =>
{
err(UNKNOWN_IOCTL)
}
_ => ok(0),
}
}
unsafe fn if_nametoindex(&self, name: *const std::ffi::c_char) -> Option<NetResult> {
if name.is_null() {
return err(libc::EFAULT);
}
let name = unsafe { std::ffi::CStr::from_ptr(name) };
match name.to_str().ok().and_then(|n| self.shared().nic(n)) {
Some(nic) => ok(nic.index as i64),
None => err(NAMETOINDEX_UNKNOWN),
}
}
unsafe fn if_indextoname(&self, index: u32, name: *mut std::ffi::c_char) -> Option<NetResult> {
let found = self.shared().nics().into_iter().find(|n| n.index == index);
let Some(nic) = found else {
return err(libc::ENXIO);
};
let bytes = nic.spec.name.as_bytes();
let n = bytes.len().min(libc::IF_NAMESIZE - 1);
unsafe {
std::ptr::copy_nonoverlapping(bytes.as_ptr(), name.cast::<u8>(), n);
*name.add(n) = 0;
}
ok(name as i64)
}
unsafe fn if_nameindex(&self) -> Option<NetResult> {
match crate::ifaddrs::nameindex(self.shared().nics()) {
Ok(ptr) => ok(ptr as i64),
Err(errno) => err(errno),
}
}
unsafe fn if_freenameindex(&self, ptr: *mut u8) -> Option<NetResult> {
crate::ifaddrs::release(ptr).then_some(NetResult::Ok(0))
}
unsafe fn getifaddrs(&self, ifap: *mut *mut u8) -> Option<NetResult> {
if ifap.is_null() {
return err(libc::EFAULT);
}
match crate::ifaddrs::getifaddrs(self.shared().nics()) {
Ok(head) => {
unsafe { *ifap = head.cast() };
ok(0)
}
Err(errno) => err(errno),
}
}
unsafe fn freeifaddrs(&self, ifa: *mut u8) -> Option<NetResult> {
crate::ifaddrs::release(ifa).then_some(NetResult::Ok(0))
}
unsafe fn poll(&self, fds: *mut u8, nfds: u64, timeout: c_int) -> Option<NetResult> {
let pfds =
unsafe { std::slice::from_raw_parts_mut(fds.cast::<libc::pollfd>(), nfds as usize) };
let watched: Vec<(c_int, i16)> = pfds.iter().map(|p| (p.fd, p.events)).collect();
if !self
.foreign_masks(watched.iter().map(|w| w.0))
.iter()
.all(|(fd, ready)| {
ready.is_some() || *fd < 0 || unsafe { libc::fcntl(*fd, libc::F_GETFD) } < 0
})
{
return None;
}
let fds: Vec<c_int> = watched.iter().map(|(fd, _)| *fd).collect();
let deadline = (timeout >= 0)
.then(|| Deadline::timeout(std::time::Duration::from_millis(timeout as u64)));
let mut waited = false;
let mut last = timeout == 0;
loop {
self.advance_connecting(Some(&fds));
let revents = self.poll_revents(&watched);
let ready = revents.iter().filter(|r| **r != 0).count();
if ready > 0 || last {
for (p, r) in pfds.iter_mut().zip(revents) {
p.revents = r;
}
if !waited {
snare_interpose::charge_latency();
}
return ok(ready as i64);
}
last = !readiness().wait_until_dynamic(
"poll",
deadline,
|keys| self.fd_interests(fds.iter().copied(), keys),
|| {
self.poll_revents(&watched).iter().any(|r| *r != 0)
|| self.connecting_due(Some(&fds))
},
);
waited = true;
}
}
unsafe fn select(
&self,
nfds: c_int,
read: *mut u8,
write: *mut u8,
except: *mut u8,
timeout: *const u8,
) -> Option<NetResult> {
if nfds < 0 || nfds as usize > libc::FD_SETSIZE {
return err(libc::EINVAL);
}
let sets = [read.cast::<libc::fd_set>(), write.cast(), except.cast()];
let mut pfds = Vec::new();
for fd in 0..nfds {
let mut events = 0;
for (set, event) in sets
.iter()
.zip([libc::POLLIN, libc::POLLOUT, libc::POLLPRI])
{
if !set.is_null() && unsafe { libc::FD_ISSET(fd, *set) } {
events |= event;
}
}
if events != 0 {
pfds.push(libc::pollfd {
fd,
events,
revents: 0,
});
}
}
if pfds
.iter()
.any(|p| unsafe { libc::fcntl(p.fd, libc::F_GETFD) } < 0)
{
return err(libc::EBADF);
}
let millis = if timeout.is_null() {
-1
} else {
let tv = unsafe { timeout.cast::<libc::timeval>().read() };
if tv.tv_sec < 0 || tv.tv_usec < 0 || tv.tv_usec >= 1_000_000 {
return err(libc::EINVAL);
}
((tv.tv_sec as u128 * 1000 + (tv.tv_usec as u128).div_ceil(1000))
.min(c_int::MAX as u128)) as c_int
};
let result =
unsafe { Net::poll(self, pfds.as_mut_ptr().cast(), pfds.len() as u64, millis) }?;
if let NetResult::Err(_) = result {
return Some(result);
}
for set in sets.into_iter().filter(|s| !s.is_null()) {
unsafe { libc::FD_ZERO(set) };
}
let mut count = 0;
for p in pfds {
for (set, event) in sets
.iter()
.zip([libc::POLLIN, libc::POLLOUT, libc::POLLPRI])
{
if !set.is_null() && p.revents & event != 0 {
unsafe { libc::FD_SET(p.fd, *set) };
count += 1;
}
}
}
ok(count)
}
unsafe fn read(&self, fd: c_int, buf: *mut u8, len: usize) -> Option<NetResult> {
match self.kind(fd)? {
Kind::Event => self.eventfd_read(fd, buf, len),
Kind::Stream | Kind::Dgram | Kind::Fresh | Kind::Listener => unsafe {
self.recv(fd, buf, len, 0)
},
Kind::Raw => self.raw_read(fd, buf, len),
_ => err(libc::EINVAL),
}
}
unsafe fn write(&self, fd: c_int, buf: *const u8, len: usize) -> Option<NetResult> {
match self.kind(fd)? {
Kind::Event => self.eventfd_write(fd, buf, len),
Kind::Stream | Kind::Dgram | Kind::Fresh | Kind::Listener => unsafe {
self.send(fd, buf, len, 0)
},
Kind::Raw => self.raw_write(fd, buf, len),
_ => err(libc::EINVAL),
}
}
#[cfg(target_os = "linux")]
unsafe fn eventfd(&self, initval: u32, flags: c_int) -> Option<NetResult> {
if flags & !(libc::EFD_NONBLOCK | libc::EFD_CLOEXEC | libc::EFD_SEMAPHORE) != 0 {
return err(libc::EINVAL);
}
let nonblocking = flags & libc::EFD_NONBLOCK != 0;
let semaphore = flags & libc::EFD_SEMAPHORE != 0;
match self.reserve_fd_flags(flags) {
Ok(fd) => {
self.socks.lock().unwrap().insert(
fd,
Sock::Event {
ready_at: self.shared().tstamp_now().mono,
ready_order: readiness_sequence(),
counter: u64::from(initval),
nonblocking,
semaphore,
writes: 0,
reads: 0,
},
);
ok(fd as i64)
}
Err(e) => err(e.raw_os_error().unwrap_or(libc::EMFILE)),
}
}
#[cfg(target_os = "linux")]
unsafe fn epoll_create1(&self, flags: c_int) -> Option<NetResult> {
if flags & !libc::EPOLL_CLOEXEC != 0 {
return err(libc::EINVAL);
}
match self.reserve_fd_flags(flags) {
Ok(fd) => {
self.socks.lock().unwrap().insert(
fd,
Sock::Epoll {
interests: Default::default(),
},
);
ok(fd as i64)
}
Err(e) => err(e.raw_os_error().unwrap_or(libc::EMFILE)),
}
}
#[cfg(target_os = "linux")]
unsafe fn epoll_ctl(
&self,
epfd: c_int,
op: c_int,
fd: c_int,
event: *const u8,
) -> Option<NetResult> {
let mut socks = self.socks.lock().unwrap();
if !socks.contains_key(&epfd) {
return None;
}
if unsafe { libc::fcntl(fd, libc::F_GETFD) } < 0 {
return err(libc::EBADF);
}
if fd == epfd {
return err(libc::EINVAL);
}
let identity = socks
.identity(fd)
.map(EpollIdentity::Local)
.unwrap_or_else(|| {
self.shared()
.sockets
.lookup_fd(fd)
.map_or(EpollIdentity::Untracked, |rec| {
EpollIdentity::Foreign(rec.id.get())
})
});
if matches!(identity, EpollIdentity::Local(id) if Some(id) == socks.identity(epfd)) {
return err(libc::EINVAL);
}
let Some(Sock::Epoll { interests }) = socks.get_mut(&epfd) else {
return err(libc::EINVAL);
};
let key = (fd, identity);
match op {
libc::EPOLL_CTL_ADD | libc::EPOLL_CTL_MOD => {
if op == libc::EPOLL_CTL_ADD && interests.contains_key(&key) {
return err(libc::EEXIST);
}
if op == libc::EPOLL_CTL_MOD && !interests.contains_key(&key) {
return err(libc::ENOENT);
}
if event.is_null() {
return err(libc::EFAULT);
}
let ev = unsafe { std::ptr::read_unaligned(event.cast::<libc::epoll_event>()) };
interests.insert(
key,
EpollInterest {
events: ev.events,
data: ev.u64,
edge: Edge::default(),
spent: false,
order: readiness_sequence(),
pending_order: None,
},
);
}
libc::EPOLL_CTL_DEL => {
if interests.remove(&key).is_none() {
return err(libc::ENOENT);
}
}
_ => return err(libc::EINVAL),
}
drop(socks);
self.bump_fds([epfd]);
ok(0)
}
#[cfg(target_os = "linux")]
unsafe fn epoll_wait(
&self,
epfd: c_int,
events: *mut u8,
maxevents: c_int,
timeout: c_int,
) -> Option<NetResult> {
if !Net::owns(self, epfd) {
return None;
}
if maxevents <= 0 {
return err(libc::EINVAL);
}
if events.is_null() {
return err(libc::EFAULT);
}
if !matches!(
self.socks.lock().unwrap().get(&epfd),
Some(Sock::Epoll { .. })
) {
return err(libc::EINVAL);
}
let deadline = (timeout >= 0)
.then(|| Deadline::timeout(std::time::Duration::from_millis(timeout as u64)));
let mut waited = false;
loop {
self.advance_connecting(None);
let ready = self.collect_ready(epfd, maxevents as usize, true);
if !ready.is_empty() {
if !waited {
snare_interpose::charge_latency();
}
for (i, (ev, data)) in ready.iter().enumerate() {
let slot = unsafe { events.cast::<libc::epoll_event>().add(i) };
unsafe {
std::ptr::write_unaligned(
slot,
libc::epoll_event {
events: *ev,
u64: *data,
},
)
};
}
return ok(ready.len() as i64);
}
let woke = readiness().wait_until_dynamic(
"epoll_wait",
deadline,
|keys| self.epoll_interests(epfd, keys),
|| !self.collect_ready(epfd, 1, false).is_empty() || self.connecting_due(None),
);
if !woke {
if timeout == 0 {
snare_interpose::charge_latency();
}
return ok(0);
}
waited = true;
}
}
#[cfg(target_os = "macos")]
unsafe fn kqueue(&self) -> Option<NetResult> {
match self.reserve_fd() {
Ok(fd) => {
self.socks.lock().unwrap().insert(
fd,
Sock::Kqueue {
state: Arc::new(Mutex::new(KqueueState::default())),
},
);
ok(fd as i64)
}
Err(e) => err(e.raw_os_error().unwrap_or(libc::EMFILE)),
}
}
#[cfg(target_os = "macos")]
unsafe fn kevent(
&self,
kq: c_int,
changelist: *const u8,
nchanges: c_int,
eventlist: *mut u8,
nevents: c_int,
timeout: *const u8,
) -> Option<NetResult> {
let state = {
let socks = self.socks.lock().unwrap();
match socks.get(&kq) {
Some(Sock::Kqueue { state }) => state.clone(),
_ => return None,
}
};
let valid_changes: Vec<bool> = {
let socks = self.socks.lock().unwrap();
(0..nchanges.max(0) as usize)
.map(|i| {
let ch = unsafe { changelist.cast::<libc::kevent>().add(i).read_unaligned() };
!matches!(ch.filter, libc::EVFILT_READ | libc::EVFILT_WRITE)
|| socks.contains_key(&(ch.ident as c_int))
})
.collect()
};
let mut receipts: Vec<libc::kevent> = Vec::new();
let mut plain_error = None;
let mut changed = false;
{
let mut guard = state.lock().unwrap();
let KqueueState {
reads,
writes,
users,
} = &mut *guard;
for (i, &valid) in valid_changes.iter().enumerate() {
let ch = unsafe { changelist.cast::<libc::kevent>().add(i).read_unaligned() };
let ident = ch.ident;
let filter = ch.filter;
let flags = ch.flags;
let fflags = ch.fflags;
let udata = ch.udata as u64;
let adding = flags & libc::EV_ADD != 0;
let deleting = flags & libc::EV_DELETE != 0;
let kept =
flags & (libc::EV_ADD | libc::EV_CLEAR | libc::EV_ONESHOT | libc::EV_DISPATCH);
let toggle = |disabled: &mut bool| {
if flags & libc::EV_DISABLE != 0 {
*disabled = true;
}
if flags & libc::EV_ENABLE != 0 {
*disabled = false;
}
};
let exists = match filter {
libc::EVFILT_READ => reads.contains_key(&(ident as c_int)),
libc::EVFILT_WRITE => writes.contains_key(&(ident as c_int)),
libc::EVFILT_USER => users.contains_key(&ident),
_ => false,
};
let change_error = if !valid {
libc::EBADF
} else if !adding && !exists {
libc::ENOENT
} else {
0
};
if change_error != 0 && flags & libc::EV_RECEIPT == 0 {
if nevents > receipts.len() as c_int {
let mut ev = ch;
ev.flags |= libc::EV_ERROR;
ev.data = change_error as _;
receipts.push(ev);
} else {
plain_error.get_or_insert(change_error);
}
}
if change_error == 0 {
changed |= matches!(
filter,
libc::EVFILT_READ | libc::EVFILT_WRITE | libc::EVFILT_USER
);
match filter {
libc::EVFILT_READ | libc::EVFILT_WRITE => {
let notes = if filter == libc::EVFILT_READ {
&mut *reads
} else {
&mut *writes
};
let fd = ident as c_int;
if deleting {
notes.remove(&fd);
} else {
if adding {
let note = notes.entry(fd).or_insert(Knote {
udata,
flags: kept,
disabled: false,
edge: Edge::default(),
});
note.udata = udata;
note.edge = Edge::default();
}
if let Some(note) = notes.get_mut(&fd) {
toggle(&mut note.disabled);
}
}
}
libc::EVFILT_USER => {
if deleting {
users.remove(&ident);
} else {
let note = users.entry(ident).or_insert(UserNote {
udata,
flags: kept,
disabled: false,
triggered: false,
});
if adding {
note.udata = udata;
}
toggle(&mut note.disabled);
if fflags & libc::NOTE_TRIGGER != 0 {
note.triggered = true;
}
}
}
_ => {}
}
}
if flags & libc::EV_RECEIPT != 0 {
let mut ev = ch;
ev.flags = flags | libc::EV_ERROR;
ev.data = change_error as _;
receipts.push(ev);
}
}
}
if changed {
self.bump_fds([kq]);
}
if let Some(errno) = plain_error {
return err(errno);
}
let cap = nevents.max(0) as usize;
if !receipts.is_empty() {
let n = receipts.len().min(cap);
for (i, ev) in receipts.iter().take(n).enumerate() {
unsafe { std::ptr::write_unaligned(eventlist.cast::<libc::kevent>().add(i), *ev) };
}
return ok(n as i64);
}
if cap == 0 {
return ok(0);
}
let deadline = unsafe { kevent_deadline(timeout) };
let zero_timeout = !timeout.is_null()
&& unsafe {
let span = timeout.cast::<libc::timespec>().read_unaligned();
span.tv_sec == 0 && span.tv_nsec == 0
};
let immediate = deadline.is_some_and(|d| d.passed());
let mut waited = false;
loop {
self.advance_connecting(None);
let ready = self.collect_kevents(kq, cap, true);
if !ready.is_empty() {
if !waited {
snare_interpose::charge_latency();
}
for (i, ev) in ready.iter().enumerate() {
unsafe {
std::ptr::write_unaligned(eventlist.cast::<libc::kevent>().add(i), *ev)
};
}
return ok(ready.len() as i64);
}
if zero_timeout {
snare_interpose::end_spin();
let ready = self.collect_kevents(kq, cap, true);
snare_interpose::charge_latency();
for (i, ev) in ready.iter().enumerate() {
unsafe {
std::ptr::write_unaligned(eventlist.cast::<libc::kevent>().add(i), *ev)
};
}
return ok(ready.len() as i64);
}
let woke = readiness().wait_until_dynamic(
"kevent",
deadline,
|keys| self.kqueue_interests(kq, keys),
|| !self.collect_kevents(kq, 1, false).is_empty() || self.connecting_due(None),
);
if !woke {
if immediate {
snare_interpose::charge_latency();
}
return ok(0);
}
waited = true;
}
}
}
impl Conn {
pub(crate) fn write_from_peer(&self, bytes: &[u8]) -> usize {
snare_interpose::real(|| self.write(End::B, bytes).unwrap_or(0))
}
pub(crate) fn peer_can_write(&self, len: usize) -> bool {
snare_interpose::real(|| {
#[cfg(target_os = "macos")]
self.progress_tcp_shutdown();
self.write_pipe(End::B).can_write(len)
})
}
pub(crate) fn read_from_peer(&self, out: &mut [u8]) -> io::Result<usize> {
snare_interpose::real(|| {
#[cfg(target_os = "macos")]
self.progress_tcp_shutdown();
match self.read_pipe(End::B).read(out) {
Read::Data(n) => {
self.took(End::B, n);
Ok(n)
}
Read::Eof => Ok(0),
Read::Reset => Err(io::ErrorKind::ConnectionReset.into()),
Read::WouldBlock | Read::Shut => Err(io::ErrorKind::WouldBlock.into()),
}
})
}
pub(crate) fn close_peer(&self) {
snare_interpose::real(|| self.close(End::B));
}
pub(crate) fn reset_peer(&self) {
snare_interpose::real(|| self.reset(End::B));
}
fn is_reset(&self) -> bool {
self.a_to_b.is_reset()
}
pub(crate) fn peer_readable(&self) -> bool {
snare_interpose::real(|| {
#[cfg(target_os = "macos")]
self.progress_tcp_shutdown();
self.read_pipe(End::B).readable_or_closed()
})
}
}
fn loopback_for(domain: c_int) -> IpAddr {
if domain == libc::AF_INET6 {
IpAddr::V6(Ipv6Addr::LOCALHOST)
} else {
IpAddr::V4(Ipv4Addr::LOCALHOST)
}
}
#[cfg(target_os = "macos")]
fn make_kevent(ident: usize, filter: i16, udata: u64) -> libc::kevent {
libc::kevent {
ident,
filter,
flags: 0,
fflags: 0,
data: 0,
udata: udata as *mut libc::c_void,
}
}
#[cfg(target_os = "macos")]
unsafe fn kevent_deadline(timeout: *const u8) -> Option<Deadline> {
if timeout.is_null() {
return None;
}
let ts = unsafe { timeout.cast::<libc::timespec>().read() };
let d = std::time::Duration::new(ts.tv_sec.max(0) as u64, ts.tv_nsec.max(0) as u32);
Some(Deadline::timeout(d))
}
unsafe fn read_int(val: *const u8, len: u32) -> c_int {
if val.is_null() || (len as usize) < size_of::<c_int>() {
return 0;
}
unsafe { val.cast::<c_int>().read_unaligned() }
}
pub(crate) unsafe fn parse_timeval(val: *const u8, len: u32) -> Option<std::time::Duration> {
if val.is_null() || (len as usize) < size_of::<libc::timeval>() {
return None;
}
let tv = unsafe { val.cast::<libc::timeval>().read_unaligned() };
let usecs = u64::try_from(tv.tv_usec).unwrap_or(0);
Some(std::time::Duration::new(
tv.tv_sec.max(0) as u64,
(usecs % 1_000_000) as u32 * 1_000,
))
}
pub(crate) unsafe fn write_timeval(d: std::time::Duration, val: *mut u8, len: *mut u32) {
if val.is_null() || len.is_null() || (unsafe { *len } as usize) < size_of::<libc::timeval>() {
return;
}
let tv = libc::timeval {
tv_sec: d.as_secs() as libc::time_t,
tv_usec: d.subsec_micros() as libc::suseconds_t,
};
unsafe {
val.cast::<libc::timeval>().write_unaligned(tv);
*len = size_of::<libc::timeval>() as u32;
}
}
#[cfg(target_os = "linux")]
const IPV6_JOIN: c_int = libc::IPV6_ADD_MEMBERSHIP;
#[cfg(not(target_os = "linux"))]
const IPV6_JOIN: c_int = libc::IPV6_JOIN_GROUP;
#[cfg(target_os = "linux")]
const IPV6_LEAVE: c_int = libc::IPV6_DROP_MEMBERSHIP;
#[cfg(not(target_os = "linux"))]
const IPV6_LEAVE: c_int = libc::IPV6_LEAVE_GROUP;
unsafe fn parse_add_membership(
level: c_int,
name: c_int,
val: *const u8,
len: u32,
) -> Option<Membership> {
if val.is_null() {
return None;
}
let len = len as usize;
if level == libc::IPPROTO_IP && name == libc::IP_ADD_MEMBERSHIP {
if len < size_of::<libc::in_addr>() {
return None;
}
let group = unsafe { val.cast::<libc::in_addr>().read_unaligned() };
let interface = (len >= 2 * size_of::<libc::in_addr>())
.then(|| unsafe { val.add(4).cast::<libc::in_addr>().read_unaligned() })
.map(|a| Ipv4Addr::from(u32::from_be(a.s_addr)))
.filter(|a| !a.is_unspecified());
let ifindex = if cfg!(target_os = "linux") && len >= 12 {
unsafe { val.add(8).cast::<c_int>().read_unaligned() }.max(0) as u32
} else {
0
};
return Some(Membership {
group: IpAddr::V4(Ipv4Addr::from(u32::from_be(group.s_addr))),
interface_addr: interface.map(IpAddr::V4),
ifindex,
});
}
if level == libc::IPPROTO_IPV6 && name == IPV6_JOIN {
if len < size_of::<libc::in6_addr>() {
return None;
}
let group = unsafe { val.cast::<libc::in6_addr>().read_unaligned() };
let ifindex = if len >= size_of::<libc::ipv6_mreq>() {
unsafe { val.add(16).cast::<u32>().read_unaligned() }
} else {
0
};
return Some(Membership {
group: IpAddr::V6(Ipv6Addr::from(group.s6_addr)),
interface_addr: None,
ifindex,
});
}
None
}
#[cfg(target_os = "macos")]
const SO_LINGER_SEC: c_int = 0x1080;
#[cfg(target_os = "linux")]
const LINGER_UNIT: Duration = Duration::from_secs(1);
#[cfg(not(target_os = "linux"))]
const LINGER_UNIT: Duration = Duration::from_millis(10);
fn linger_unit(name: c_int) -> Option<Duration> {
match name {
libc::SO_LINGER => Some(LINGER_UNIT),
#[cfg(target_os = "macos")]
SO_LINGER_SEC => Some(Duration::from_secs(1)),
_ => None,
}
}
unsafe fn parse_linger(val: *const u8, len: u32, unit: Duration) -> Option<Duration> {
if val.is_null() || (len as usize) < size_of::<libc::linger>() {
return None;
}
let l = unsafe { val.cast::<libc::linger>().read_unaligned() };
(l.l_onoff != 0).then(|| unit * l.l_linger.max(0) as u32)
}
unsafe fn write_linger(linger: Option<Duration>, unit: Duration, val: *mut u8, len: *mut u32) {
if val.is_null() || len.is_null() || (unsafe { *len } as usize) < size_of::<libc::linger>() {
return;
}
let l = libc::linger {
l_onoff: linger.is_some() as c_int,
l_linger: linger.map_or(0, |d| (d.as_nanos() / unit.as_nanos()) as c_int),
};
unsafe {
val.cast::<libc::linger>().write_unaligned(l);
*len = size_of::<libc::linger>() as u32;
}
}
unsafe fn parse_addr(ptr: *const u8, len: u32) -> Option<SocketAddr> {
if ptr.is_null() || (len as usize) < size_of::<libc::sockaddr>() {
return None;
}
let family = unsafe { (*ptr.cast::<libc::sockaddr>()).sa_family } as c_int;
match family {
libc::AF_INET => {
if (len as usize) < size_of::<libc::sockaddr_in>() {
return None;
}
let sa = unsafe { std::ptr::read_unaligned(ptr.cast::<libc::sockaddr_in>()) };
let ip = Ipv4Addr::from(u32::from_be(sa.sin_addr.s_addr));
Some(SocketAddr::V4(SocketAddrV4::new(
ip,
u16::from_be(sa.sin_port),
)))
}
libc::AF_INET6 => {
if (len as usize) < size_of::<libc::sockaddr_in6>() {
return None;
}
let sa = unsafe { std::ptr::read_unaligned(ptr.cast::<libc::sockaddr_in6>()) };
let ip = Ipv6Addr::from(sa.sin6_addr.s6_addr);
Some(SocketAddr::V6(SocketAddrV6::new(
ip,
u16::from_be(sa.sin6_port),
sa.sin6_flowinfo,
sa.sin6_scope_id,
)))
}
_ => None,
}
}
const UNNAMED: SocketAddr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0));
#[cfg(target_os = "linux")]
const PAIR_PEER_GONE: c_int = libc::ECONNREFUSED;
#[cfg(not(target_os = "linux"))]
const PAIR_PEER_GONE: c_int = libc::ECONNRESET;
unsafe fn write_unnamed(out: *mut u8, out_len: *mut u32) {
if out.is_null() || out_len.is_null() {
return;
}
#[cfg(target_os = "linux")]
let bytes: &[u8] = &(libc::AF_UNIX as u16).to_ne_bytes();
#[cfg(not(target_os = "linux"))]
let bytes: &[u8] = &{
let mut b = [0u8; 16];
b[0] = 16;
b[1] = libc::AF_UNIX as u8;
b
};
unsafe {
let n = (*out_len as usize).min(bytes.len());
std::ptr::copy_nonoverlapping(bytes.as_ptr(), out, n);
*out_len = bytes.len() as u32;
}
}
unsafe fn write_addr(addr: SocketAddr, out: *mut u8, out_len: *mut u32) {
if out.is_null() || out_len.is_null() {
return;
}
let cap = unsafe { *out_len } as usize;
match addr {
SocketAddr::V4(v4) => {
let mut sa: libc::sockaddr_in = unsafe { std::mem::zeroed() };
sa.sin_family = libc::AF_INET as libc::sa_family_t;
sa.sin_port = v4.port().to_be();
sa.sin_addr.s_addr = u32::from(*v4.ip()).to_be();
unsafe { copy_out(&sa, out, out_len, cap) };
}
SocketAddr::V6(v6) => {
let mut sa: libc::sockaddr_in6 = unsafe { std::mem::zeroed() };
sa.sin6_family = libc::AF_INET6 as libc::sa_family_t;
sa.sin6_port = v6.port().to_be();
sa.sin6_addr.s6_addr = v6.ip().octets();
unsafe { copy_out(&sa, out, out_len, cap) };
}
}
}
unsafe fn copy_out<T>(src: &T, out: *mut u8, out_len: *mut u32, cap: usize) {
let size = size_of::<T>();
let n = size.min(cap);
unsafe {
std::ptr::copy_nonoverlapping(src as *const T as *const u8, out, n);
*out_len = size as u32;
}
}
pub(crate) unsafe fn write_opt(value: i32, val: *mut u8, len: *mut u32) {
if val.is_null() || len.is_null() {
return;
}
let cap = unsafe { *len } as usize;
let n = size_of::<i32>().min(cap);
unsafe {
std::ptr::copy_nonoverlapping(&value as *const i32 as *const u8, val, n);
*len = size_of::<i32>() as u32;
}
}
enum Progress {
Pending(Deadline),
Settled(Outcome, Arc<SockRec>),
Idle,
}
impl Fabric {
fn listener_for(&self, server: SocketAddr, host_local: bool) -> Option<Arc<Listener>> {
let listeners = self.regs.listeners.lock().unwrap();
listeners
.get(&server)
.or_else(|| {
host_local
.then(|| {
listeners.get(&SocketAddr::new(
unspecified_like(server.ip()),
server.port(),
))
})
.flatten()
})
.cloned()
}
fn client_addr(
&self,
server: SocketAddr,
bound: Option<SocketAddr>,
sender: &Sender,
) -> SocketAddr {
let src = sender
.source(SocketAddr::new(unspecified_like(server.ip()), 0))
.ip();
match bound {
Some(local) if !local.ip().is_unspecified() => local,
Some(local) => SocketAddr::new(src, local.port()),
None => {
let socks = self.socks.lock().unwrap();
let local = self.tcp_bind_addr(&socks, SocketAddr::new(src, 0), false);
local.unwrap_or(SocketAddr::new(src, 0))
}
}
}
fn establish(
&self,
rec: &Arc<SockRec>,
server: SocketAddr,
client: SocketAddr,
sender: &Sender,
server_in: bool,
) -> Arc<Conn> {
let hop = self.shared().tcp_hop(sender, server.ip());
let via = hop.as_ref().map(|hop| hop.link.clone());
let ends = |from, to| {
Some(Ends {
shared: self.regs.shared.clone(),
from,
to,
})
};
let conn = Arc::new(Conn {
a_to_b: Pipe::new(
via.clone(),
ends(client, server),
self.shared().domain_key(),
),
b_to_a: Pipe::new(via, ends(server, client), self.shared().domain_key()),
client,
server,
regs: self.regs.clone(),
hop,
pair: false,
tap: self.shared().tcp_tap(sender, client, server, server_in),
stamps: Default::default(),
write_delays: Default::default(),
#[cfg(target_os = "macos")]
mac_shutdown: Default::default(),
});
#[cfg(target_os = "linux")]
{
let topo = self.shared().topo();
let mtu = topo
.mtu_of(conn.hop.as_ref().map_or(topo.loopback_index(), |h| h.index))
.unwrap_or(1500);
conn.a_to_b.set_mtu(mtu);
conn.b_to_a.set_mtu(mtu);
for end in [End::A, End::B] {
let weak = Arc::downgrade(&conn);
conn.write_pipe(end)
.on_transmit(Box::new(move |bytes, arrival, mss| {
if let Some(conn) = weak.upgrade() {
conn.record_write(end, bytes, arrival, Some(mss), true);
}
}));
}
}
rec.set_ends(client, server);
rec.note_conn_nic(sender.egress_name());
if !server_in {
crate::netstats::bump(&self.shared().stats.tcp(server.ip()).passive_opens);
}
conn.b_to_a.read_by(rec);
conn.a_to_b.write_by(rec);
#[cfg(target_os = "macos")]
conn.attach_tcp_probe(End::A, rec);
conn
}
fn await_connect(&self, fd: c_int) -> Option<NetResult> {
loop {
match self.advance_connect(fd) {
Progress::Pending(next) => {
readiness().wait_until_dynamic(
"connect",
Some(next),
|keys| self.fd_interests([fd], keys),
|| self.connect_due(fd),
);
}
Progress::Settled(Outcome::Connected, _) => return ok(0),
Progress::Settled(Outcome::Failed(errno), rec) => {
rec.take_error();
return err(errno);
}
Progress::Idle => {
return match self.socks.lock().unwrap().get(&fd) {
Some(Sock::Stream { .. }) => ok(0),
Some(Sock::Fresh { rec, .. }) => {
err(rec.take_error().unwrap_or(libc::ECONNABORTED))
}
_ => err(libc::EBADF),
};
}
}
}
}
fn advance_connect(&self, fd: c_int) -> Progress {
let mut socks = self.socks.lock().unwrap();
let Some(Sock::Connecting {
local,
rec,
dest,
attempt,
..
}) = socks.get(&fd)
else {
return Progress::Idle;
};
let (local, rec, dest, attempt) = (*local, rec.clone(), *dest, attempt.clone());
let client = attempt.lock().unwrap().client;
let mut listener = None;
let mut admission = None;
let step = attempt.lock().unwrap().poll(|| {
let station = self.regs.station_at(dest.ip());
listener = self.listener_for(dest, self.shared().dest_is_host(dest.ip(), station));
let syn = self.shared().syn_probe(dest, listener.is_some(), station);
if matches!(syn, Syn::Accept) {
admission = listener.as_ref().and_then(Listener::reserve);
if admission.is_none() {
return Syn::Silent;
}
}
syn
});
let outcome = match step {
Ok(next) => return Progress::Pending(next),
Err(outcome) => outcome,
};
let conn = match (outcome, listener) {
(Outcome::Connected, Some(listener)) => {
let station = self.regs.station_at(dest.ip());
self.shared()
.route_send(&rec.view(local), dest, Op::Connect, station, true)
.map(|sender| {
let conn = self.establish(&rec, dest, client, &sender, listener.tester);
if let Some(lrec) = listener.rec.as_ref().and_then(std::sync::Weak::upgrade)
{
conn.a_to_b.pending_reader(lrec.state().buf);
}
(conn, listener)
})
}
(Outcome::Connected, None) => Err(crate::faults::code::ECONNREFUSED),
(Outcome::Failed(errno), _) => Err(errno),
};
let Some(Sock::Connecting {
domain,
local,
nonblocking,
rec: settled_rec,
..
}) = socks.get(&fd)
else {
return Progress::Idle;
};
let settled = match &conn {
Ok((conn, _)) => Sock::Stream {
conn: conn.clone(),
end: End::A,
nonblocking: *nonblocking,
rec: settled_rec.clone(),
},
Err(_) => Sock::Fresh {
domain: *domain,
local: *local,
nonblocking: *nonblocking,
rec: settled_rec.clone(),
},
};
socks.insert(fd, settled);
let mut keys = crate::readiness::WakeKeys::default();
keys.push(rec.wake_key());
let outcome = match conn {
Ok((conn, listener)) => {
attempt.lock().unwrap().established(conn.tap.as_ref());
if let Some(key) = listener.wake_key() {
keys.push(key);
}
admission
.expect("accepted SYN reserves a queue entry")
.queue(conn);
Outcome::Connected
}
Err(errno) => {
rec.state().tcp_failed = true;
rec.set_pending_error(errno, None);
Outcome::Failed(errno)
}
};
drop(socks);
self.shared().connect_settled(dest, outcome, true);
self.shared().bump_keys(keys.as_slice());
Progress::Settled(outcome, rec)
}
fn advance_connecting(&self, fds: Option<&[c_int]>) {
let connecting: Vec<c_int> = self
.socks
.lock()
.unwrap()
.iter()
.filter(|(fd, sock)| {
matches!(sock, Sock::Connecting { .. }) && fds.is_none_or(|fds| fds.contains(fd))
})
.map(|(fd, _)| *fd)
.collect();
for fd in connecting {
self.advance_connect(fd);
}
}
fn connect_due(&self, fd: c_int) -> bool {
match self.socks.lock().unwrap().get(&fd) {
Some(Sock::Connecting { attempt, .. }) => attempt.lock().unwrap().due(),
_ => true,
}
}
fn connecting_due(&self, fds: Option<&[c_int]>) -> bool {
self.socks
.lock()
.unwrap()
.iter()
.any(|(fd, sock)| match sock {
Sock::Connecting { attempt, .. } => {
fds.is_none_or(|fds| fds.contains(fd)) && attempt.lock().unwrap().due()
}
_ => false,
})
}
}
#[derive(Clone, Copy, PartialEq)]
enum Kind {
Fresh,
Stream,
Listener,
Dgram,
Event,
Epoll,
Raw,
#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
Kqueue,
}
pub(crate) fn readiness_sequence() -> u64 {
static SEQUENCE: AtomicU64 = AtomicU64::new(1);
SEQUENCE.fetch_add(1, Ordering::Relaxed)
}
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), allow(dead_code))]
fn readiness_order(sock: &Sock, fallback: u64) -> (Duration, u64) {
match sock {
Sock::Dgram { queue, .. } | Sock::UnixDgram { queue, .. } => queue
.readiness_metadata()
.map_or((Duration::MAX, fallback), |(at, sequence, _)| {
(at, sequence)
}),
Sock::Stream { conn, end, .. } => {
let writer = if *end == End::A { End::B } else { End::A };
let at = conn.stamps[writer.index()]
.lock()
.unwrap()
.front()
.map_or(Duration::ZERO, |(_, stamp)| stamp.mono);
(at, conn.read_pipe(*end).read_order().max(fallback))
}
Sock::Listener { listener, .. } => (
Duration::from_nanos(listener.ready_at.load(Ordering::Relaxed)),
listener.ready_order.load(Ordering::Relaxed),
),
Sock::Event {
ready_order,
ready_at,
..
} => (*ready_at, *ready_order),
_ => (Duration::ZERO, fallback),
}
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
const EPOLLIN: u32 = 0x001;
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
const EPOLLOUT: u32 = 0x004;
const EPOLLERR: u32 = 0x008;
const EPOLLHUP: u32 = 0x010;
const EPOLLRDHUP: u32 = 0x2000;
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn ready_mask(sock: &Sock) -> u32 {
#[cfg(target_os = "macos")]
if let Sock::Stream { conn, .. } = sock {
conn.progress_tcp_shutdown();
}
let mask = base_mask(sock);
match sock {
Sock::Fresh { rec, .. } if rec.state().tcp_failed && cfg!(target_os = "macos") => mask,
Sock::Stream { rec, .. } | Sock::Dgram { rec, .. } | Sock::Fresh { rec, .. }
if rec.peek_error().is_some() =>
{
mask | if cfg!(target_os = "linux") {
EPOLLERR
} else {
EPOLLIN
}
}
#[cfg(target_os = "linux")]
Sock::Stream { rec, .. } | Sock::Dgram { rec, .. }
if crate::tstamp::tx_report_ready(rec) || rec.icmp_report_ready() =>
{
mask | EPOLLERR
}
_ => mask,
}
}
fn foreign_ready(rec: &SockRec) -> Option<(u32, Wakes)> {
let (readable, landed) = rec.dgram_readiness()?;
let mut mask = EPOLLOUT;
if readable {
mask |= EPOLLIN;
}
if rec.peek_error().is_some() {
mask |= if cfg!(target_os = "linux") {
EPOLLERR
} else {
EPOLLIN
};
}
#[cfg(target_os = "linux")]
if crate::tstamp::tx_report_ready(rec) || rec.icmp_report_ready() {
mask |= EPOLLERR;
}
Some((
mask,
Wakes {
read: landed,
write: 0,
error: rec.error_wakes(),
},
))
}
fn base_mask(sock: &Sock) -> u32 {
let (epollin, epollout) = (EPOLLIN, EPOLLOUT);
match sock {
Sock::Stream { conn, end, .. } => {
let mut m = 0;
if conn.write_pipe(*end).writable() {
m |= epollout;
}
let read = conn.read_pipe(*end);
let write = conn.write_pipe(*end);
if read.readable_or_closed() {
m |= epollin;
}
if read.read_eof() {
m |= EPOLLRDHUP;
}
#[cfg(target_os = "macos")]
let hung_up = write.reader_gone() || (read.read_eof() && write.is_write_closed());
#[cfg(not(target_os = "macos"))]
let hung_up = read.read_eof() && write.is_closed();
if hung_up {
m |= EPOLLHUP;
if cfg!(target_os = "macos") {
m &= !epollout;
}
}
m
}
Sock::Dgram { queue, .. } | Sock::UnixDgram { queue, .. } => {
let mut m = epollout;
if queue.has(|_| true) {
m |= epollin;
}
m
}
Sock::Event { counter, .. } => {
let mut m = if *counter < u64::MAX - 1 { epollout } else { 0 };
if *counter > 0 {
m |= epollin;
}
m
}
Sock::Raw { rx, .. } => {
let mut m = epollout;
if !rx.is_empty() {
m |= epollin;
}
m
}
Sock::Listener { listener, .. } if listener.has_pending() => epollin,
Sock::Fresh { rec, .. } if rec.state().tcp_failed => {
epollin
| EPOLLHUP
| if cfg!(target_os = "linux") {
epollout
} else {
0
}
}
Sock::Fresh { rec, .. } if rec.peek_error().is_some() => epollout | EPOLLERR | EPOLLHUP,
_ => 0,
}
}
fn poll_bits(mask: u32, events: i16) -> i16 {
let mut r = 0;
if mask & EPOLLIN != 0 && events & libc::POLLIN != 0 {
r |= libc::POLLIN;
}
if mask & EPOLLOUT != 0 && events & libc::POLLOUT != 0 {
r |= libc::POLLOUT;
}
#[cfg(target_os = "linux")]
if mask & EPOLLRDHUP != 0 && events & libc::POLLRDHUP != 0 {
r |= libc::POLLRDHUP;
}
if mask & EPOLLERR != 0 {
r |= libc::POLLERR;
}
if mask & EPOLLHUP != 0 {
r |= libc::POLLHUP;
}
r
}
#[derive(Clone, Copy, Default)]
struct Wakes {
read: u64,
write: u64,
error: u64,
}
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), allow(dead_code))]
fn wakes(sock: &Sock) -> Wakes {
let mut wakes = match sock {
Sock::Stream { conn, end, .. } => Wakes {
read: conn.read_pipe(*end).read_wakes(),
write: conn.write_pipe(*end).write_wakes(),
..Wakes::default()
},
Sock::Dgram { queue, .. } | Sock::UnixDgram { queue, .. } => Wakes {
read: queue.landed(),
write: 0,
..Wakes::default()
},
Sock::Event { writes, reads, .. } => Wakes {
read: *writes,
write: *reads,
..Wakes::default()
},
Sock::Raw { arrived, .. } => Wakes {
read: *arrived,
write: 0,
..Wakes::default()
},
Sock::Listener { listener, .. } => Wakes {
read: listener.arrivals.load(Ordering::Relaxed),
write: 0,
..Wakes::default()
},
_ => Wakes::default(),
};
wakes.error = sock.rec().map_or(0, |rec| rec.error_wakes());
wakes
}
#[derive(Clone, Copy, Default)]
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), allow(dead_code))]
struct Edge {
seen: u32,
wakes: Wakes,
}
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), allow(dead_code))]
impl Edge {
fn pending(&mut self, ready: u32, now: Wakes) -> bool {
self.seen &= ready;
ready & !self.seen != 0
|| (ready & EPOLLIN != 0 && now.read != self.wakes.read)
|| (ready & EPOLLOUT != 0 && now.write != self.wakes.write)
|| (ready
& if cfg!(target_os = "linux") {
EPOLLERR
} else {
EPOLLIN | EPOLLOUT
}
!= 0
&& now.error != self.wakes.error)
}
fn report(&mut self, ready: u32, now: Wakes) {
self.seen = ready;
self.wakes = now;
}
}
impl Sock {
fn rec(&self) -> Option<&Arc<SockRec>> {
match self {
Sock::Fresh { rec, .. }
| Sock::Connecting { rec, .. }
| Sock::Stream { rec, .. }
| Sock::Listener { rec, .. }
| Sock::Dgram { rec, .. }
| Sock::UnixDgram { rec, .. }
| Sock::Raw { rec, .. } => Some(rec),
_ => None,
}
}
}
fn tcp_bind_conflict(held: (SocketAddr, bool, bool), want: SocketAddr, reuse: bool) -> bool {
let (held, held_reuse, held_listening) = held;
if held.port() != want.port() || held.is_ipv4() != want.is_ipv4() {
return false;
}
let exact = held.ip() == want.ip();
if !exact && !held.ip().is_unspecified() && !want.ip().is_unspecified() {
return false;
}
if cfg!(target_os = "linux") {
!(reuse && held_reuse && !held_listening)
} else {
exact || !reuse
}
}
impl Fabric {
fn tcp_bind_addr(
&self,
socks: &Sockets,
mut want: SocketAddr,
reuse: bool,
) -> Option<SocketAddr> {
let mut held: Vec<(SocketAddr, bool, bool)> = socks
.values()
.filter_map(|sock| match sock {
Sock::Fresh {
local: Some(local),
rec,
..
}
| Sock::Connecting {
local: Some(local),
rec,
..
} => Some((*local, rec.opts().reuseaddr, false)),
Sock::Listener { listener, rec, .. } => {
Some((listener.addr, rec.opts().reuseaddr, true))
}
Sock::Stream { conn, end, rec, .. } if !conn.pair => {
Some((conn.local(*end), rec.opts().reuseaddr, false))
}
_ => None,
})
.collect();
held.extend(
self.regs
.listeners
.lock()
.unwrap()
.keys()
.map(|a| (*a, false, true)),
);
let free = |addr: SocketAddr| !held.iter().any(|&h| tcp_bind_conflict(h, addr, reuse));
if want.port() != 0 {
return free(want).then_some(want);
}
let port = (0..=u16::MAX - crate::scope::EPHEMERAL_FIRST)
.map(|_| self.shared().ephemeral_port())
.find(|&port| free(SocketAddr::new(want.ip(), port)))?;
want.set_port(port);
Some(want)
}
fn kind(&self, fd: c_int) -> Option<Kind> {
let socks = self.socks.lock().unwrap();
Some(match socks.get(&fd)? {
Sock::Fresh { .. } | Sock::Connecting { .. } => Kind::Fresh,
Sock::Stream { .. } => Kind::Stream,
Sock::Listener { .. } => Kind::Listener,
Sock::Dgram { .. } | Sock::UnixDgram { .. } => Kind::Dgram,
Sock::Event { .. } => Kind::Event,
Sock::Epoll { .. } => Kind::Epoll,
Sock::Raw { .. } => Kind::Raw,
Sock::Kqueue { .. } => Kind::Kqueue,
})
}
fn is_nonblocking(&self, fd: c_int) -> bool {
let socks = self.socks.lock().unwrap();
matches!(
socks.get(&fd),
Some(Sock::Event {
nonblocking: true,
..
}) | Some(Sock::Stream {
nonblocking: true,
..
}) | Some(Sock::Dgram {
nonblocking: true,
..
}) | Some(Sock::UnixDgram {
nonblocking: true,
..
}) | Some(Sock::Raw {
nonblocking: true,
..
})
)
}
fn poll_revents(&self, watched: &[(c_int, i16)]) -> Vec<i16> {
let foreign: HashMap<c_int, Option<(u32, Wakes)>> = self
.foreign_masks(watched.iter().map(|w| w.0))
.into_iter()
.collect();
let socks = self.socks.lock().unwrap();
watched
.iter()
.map(|(fd, events)| match socks.get(fd) {
Some(s) => poll_bits(ready_mask(s), *events),
None if *fd < 0 => 0,
None => foreign
.get(fd)
.copied()
.flatten()
.map_or(libc::POLLNVAL, |(mask, _)| poll_bits(mask, *events)),
})
.collect()
}
fn foreign_masks(
&self,
fds: impl Iterator<Item = c_int>,
) -> Vec<(c_int, Option<(u32, Wakes)>)> {
let others: Vec<c_int> = {
let socks = self.socks.lock().unwrap();
fds.filter(|fd| !socks.contains_key(fd)).collect()
};
others
.into_iter()
.map(|fd| {
let rec = self.shared().sockets.lookup_fd(fd);
(fd, rec.as_deref().and_then(foreign_ready))
})
.collect()
}
#[cfg(target_os = "linux")]
fn collect_ready(&self, epfd: c_int, max: usize, take: bool) -> Vec<(u32, u64)> {
let interested: Vec<(c_int, EpollIdentity)> = {
let socks = self.socks.lock().unwrap();
let Some(Sock::Epoll { interests }) = socks.get(&epfd) else {
return Vec::new();
};
interests
.keys()
.filter(|key| !matches!(key.1, EpollIdentity::Local(_)))
.copied()
.collect()
};
let foreign: HashMap<_, (u32, Wakes)> = interested
.into_iter()
.filter_map(|key| {
let rec = match key.1 {
EpollIdentity::Foreign(id) => self.shared().sockets.lookup_raw_id(id),
EpollIdentity::Untracked => self.shared().sockets.lookup_fd(key.0),
EpollIdentity::Local(_) => None,
}?;
Some((key, foreign_ready(&rec)?))
})
.collect();
let mut socks = self.socks.lock().unwrap();
let Some(Sock::Epoll { interests }) = socks.get(&epfd) else {
return Vec::new();
};
let watched: Vec<_> = interests
.iter()
.filter(|(_, i)| !i.spent)
.map(|(key, i)| (*key, i.events, i.data, i.edge, i.order, i.pending_order))
.collect();
let mut pending = Vec::new();
let mut seen = Vec::new();
let mut gone = Vec::new();
for (key, events, data, mut edge, order, pending_order) in watched {
let sock = match key.1 {
EpollIdentity::Local(id) => socks.get_identity(id),
EpollIdentity::Untracked => socks.get(&key.0),
EpollIdentity::Foreign(_) => None,
};
let (mask, now, arrival) = match sock {
Some(sock) => {
let mask = ready_mask(sock);
(mask, wakes(sock), readiness_order(sock, order))
}
None => match foreign.get(&key) {
Some(&(mask, now)) => (mask, now, (Duration::ZERO, order)),
None => {
gone.push(key);
continue;
}
},
};
let got = mask & (events | EPOLLERR | EPOLLHUP);
let edged = edge.pending(got, now);
let fire = got != 0 && (events & libc::EPOLLET as u32 == 0 || edged);
if fire {
pending.push((pending_order.unwrap_or(arrival), key, got, data, now));
}
seen.push((key, edge, fire.then_some(pending_order.unwrap_or(arrival))));
}
pending.sort_by_key(|p| (p.0, p.1.0));
let mut out = Vec::new();
if let Some(Sock::Epoll { interests }) = socks.get_mut(&epfd) {
for key in gone {
interests.remove(&key);
}
for (key, edge, pending_order) in seen {
if let Some(i) = interests.get_mut(&key) {
i.edge = edge;
i.pending_order = pending_order;
}
}
for (_, key, got, data, now) in pending.into_iter().take(max) {
out.push((got, data));
if take && let Some(i) = interests.get_mut(&key) {
i.edge.report(got, now);
i.spent |= i.events & libc::EPOLLONESHOT as u32 != 0;
i.pending_order = if i.events & libc::EPOLLET as u32 == 0 {
Some((Duration::MAX, readiness_sequence()))
} else {
None
};
}
}
}
out
}
#[cfg(target_os = "macos")]
fn collect_kevents(&self, kq: c_int, max: usize, take: bool) -> Vec<libc::kevent> {
let state = {
let socks = self.socks.lock().unwrap();
match socks.get(&kq) {
Some(Sock::Kqueue { state }) => state.clone(),
_ => return Vec::new(),
}
};
type Watch = (c_int, u64, u16, Edge);
let live = |notes: &HashMap<c_int, Knote>| -> Vec<Watch> {
notes
.iter()
.filter(|(_, n)| !n.disabled)
.map(|(fd, n)| (*fd, n.udata, n.flags, n.edge))
.collect()
};
let g = state.lock().unwrap();
let reads = live(&g.reads);
let writes = live(&g.writes);
let users_ready: Vec<(usize, u64)> = g
.users
.iter()
.filter(|(_, n)| n.triggered && !n.disabled)
.map(|(id, n)| (*id, n.udata))
.collect();
drop(g);
let socks = self.socks.lock().unwrap();
let mut out = Vec::new();
let mut seen: Vec<(i16, c_int, Edge, bool)> = Vec::new();
for (filter, watched) in [(libc::EVFILT_READ, reads), (libc::EVFILT_WRITE, writes)] {
if out.len() >= max {
break;
}
let bit = if filter == libc::EVFILT_READ {
EPOLLIN
} else {
EPOLLOUT
};
let mut pending = Vec::new();
for (fd, udata, flags, mut edge) in watched {
let Some(sock) = socks.get(&fd) else {
continue;
};
let mask = ready_mask(sock);
let got = if matches!(sock, Sock::Fresh { rec, .. } if rec.state().tcp_failed)
|| (filter == libc::EVFILT_WRITE
&& matches!(sock, Sock::Stream { conn, end, .. }
if conn.write_pipe(*end).is_write_closed()
|| conn.write_pipe(*end).reader_gone()))
{
bit
} else {
mask & bit
};
let now = wakes(sock);
let edged = edge.pending(got, now);
let fire = got != 0 && (flags & libc::EV_CLEAR == 0 || edged);
let seen_index = seen.len();
seen.push((filter, fd, edge, false));
if fire {
pending.push((
readiness_order(sock, fd as u64),
fd,
udata,
flags,
got,
now,
seen_index,
));
}
}
pending.sort_by_key(|p| (p.0, p.1));
for (_, fd, udata, flags, got, now, seen_index) in
pending.into_iter().take(max - out.len())
{
if let Some(sock) = socks.get(&fd) {
let mut ev = make_kevent(fd as usize, filter, udata);
ev.flags = flags;
if let Sock::Fresh { rec, .. } = sock
&& rec.state().tcp_failed
{
ev.flags |= libc::EV_EOF;
ev.fflags = rec.peek_error().unwrap_or(0) as u32;
}
if filter == libc::EVFILT_READ {
if let Sock::Stream { conn, end, rec, .. } = sock {
ev.data = conn.read_pipe(*end).queued_bytes() as _;
if conn.read_pipe(*end).read_eof() {
ev.flags |= libc::EV_EOF;
ev.fflags = rec.peek_error().unwrap_or(0) as u32;
}
}
if let Sock::Dgram { queue, .. } | Sock::UnixDgram { queue, .. } = sock {
ev.data = queue.readiness_metadata().map_or(0, |(_, _, len)| len) as _;
}
if let Sock::Listener { listener, .. } = sock {
ev.data =
snare_interpose::real(|| listener.pending.lock().unwrap().len())
as _;
}
} else {
if let Sock::Stream { conn, end, rec, .. } = sock {
ev.data = conn.write_pipe(*end).room().unwrap_or(8192) as _;
if conn.write_pipe(*end).is_write_closed()
|| conn.write_pipe(*end).reader_gone()
{
ev.flags |= libc::EV_EOF;
ev.fflags = rec.peek_error().unwrap_or(0) as u32;
}
}
if let Sock::Fresh { rec, .. } = sock {
ev.data = rec.state().buf.tcp_send_space().min(2048) as _;
}
if let Sock::Fresh { rec, .. } = sock
&& let Some(errno) = rec.peek_error()
{
ev.flags |= libc::EV_EOF;
ev.fflags = errno as u32;
}
}
out.push(ev);
if take {
seen[seen_index].2.report(got, now);
seen[seen_index].3 = true;
}
}
}
}
drop(socks);
let mut users_taken = Vec::new();
for (id, udata) in users_ready {
if out.len() >= max {
break;
}
let mut ev = make_kevent(id, libc::EVFILT_USER, udata);
ev.flags = state.lock().unwrap().users.get(&id).map_or(0, |n| n.flags);
out.push(ev);
users_taken.push(id);
}
let mut g = state.lock().unwrap();
for (filter, fd, edge, reported) in seen {
let notes = if filter == libc::EVFILT_READ {
&mut g.reads
} else {
&mut g.writes
};
let Some(note) = notes.get_mut(&fd) else {
continue;
};
note.edge = edge;
if reported {
if note.flags & libc::EV_ONESHOT != 0 {
notes.remove(&fd);
} else if note.flags & libc::EV_DISPATCH != 0 {
note.disabled = true;
}
}
}
if take {
for id in users_taken {
let Some(note) = g.users.get_mut(&id) else {
continue;
};
if note.flags & libc::EV_ONESHOT != 0 {
g.users.remove(&id);
continue;
}
if note.flags & libc::EV_DISPATCH != 0 {
note.disabled = true;
}
if note.flags & libc::EV_CLEAR != 0 {
note.triggered = false;
}
}
}
out
}
fn eventfd_peek(&self, fd: c_int) -> u64 {
let socks = self.socks.lock().unwrap();
match socks.get(&fd) {
Some(Sock::Event { counter, .. }) => *counter,
_ => 0,
}
}
fn eventfd_take(&self, fd: c_int) -> Option<u64> {
let mut socks = self.socks.lock().unwrap();
let Some(Sock::Event {
counter,
semaphore,
reads,
..
}) = socks.get_mut(&fd)
else {
return None;
};
if *counter == 0 {
return None;
}
*reads += 1;
let value = if *semaphore {
*counter -= 1;
1
} else {
std::mem::take(counter)
};
Some(value)
}
fn raw_pop(&self, fd: c_int) -> Option<Vec<u8>> {
let mut socks = self.socks.lock().unwrap();
match socks.get_mut(&fd) {
Some(Sock::Raw { rx, .. }) => rx.pop_front(),
_ => None,
}
}
fn raw_has(&self, fd: c_int) -> bool {
let socks = self.socks.lock().unwrap();
matches!(socks.get(&fd), Some(Sock::Raw { rx, .. }) if !rx.is_empty())
}
fn raw_read(&self, fd: c_int, buf: *mut u8, len: usize) -> Option<NetResult> {
let frame = if let Some(f) = self.raw_pop(fd) {
f
} else if self.is_nonblocking(fd) {
return would_block();
} else if readiness().wait_until_dynamic(
"raw recv",
self.recv_deadline(fd),
|keys| self.fd_interests([fd], keys),
|| self.raw_has(fd),
) {
self.raw_pop(fd)?
} else {
return err(libc::EAGAIN); };
let n = len.min(frame.len());
unsafe { std::ptr::copy_nonoverlapping(frame.as_ptr(), buf, n) };
ok(n as i64)
}
fn raw_write(&self, fd: c_int, buf: *const u8, len: usize) -> Option<NetResult> {
let frame = unsafe { std::slice::from_raw_parts(buf, len) }.to_vec();
let (ifindex, rec) = match self.socks.lock().unwrap().get(&fd) {
Some(Sock::Raw {
ifindex: Some(i),
rec,
..
}) => (*i, rec.clone()),
_ => return err(libc::EINVAL),
};
let sent = self.shared().raw_send(ifindex, len);
if sent.is_ok() {
self.shared().capture_l2(ifindex, &frame);
}
match sent {
Err(errno) => return err(errno),
Ok(false) => {
rec.count_sent();
return ok(len as i64);
}
Ok(true) => rec.count_sent(),
}
let mut keys = crate::readiness::WakeKeys::default();
keys.push(rec.wake_key());
{
let mut socks = self.socks.lock().unwrap();
for (other, sock) in socks.iter_mut() {
if *other == fd
|| sock
.rec()
.is_some_and(|other_rec| Arc::ptr_eq(other_rec, &rec))
{
continue;
}
if let Sock::Raw {
ifindex: Some(oi),
rx,
arrived,
rec,
..
} = sock
&& *oi == ifindex
{
rx.push_back(frame.clone());
*arrived += 1;
rec.count_delivered(frame.len());
keys.push(rec.wake_key());
}
}
}
self.shared().bump_keys(keys.as_slice());
ok(len as i64)
}
fn eventfd_read(&self, fd: c_int, buf: *mut u8, len: usize) -> Option<NetResult> {
if len < 8 {
return err(libc::EINVAL);
}
let value = if let Some(v) = self.eventfd_take(fd) {
v
} else if self.is_nonblocking(fd) {
return would_block();
} else if readiness().wait_until_dynamic(
"eventfd",
None,
|keys| self.fd_interests([fd], keys),
|| self.eventfd_peek(fd) > 0,
) {
self.eventfd_take(fd).unwrap_or(0)
} else {
return err(libc::EAGAIN); };
unsafe { std::ptr::copy_nonoverlapping(value.to_ne_bytes().as_ptr(), buf, 8) };
self.bump_fds([fd]);
ok(8)
}
fn eventfd_write(&self, fd: c_int, buf: *const u8, len: usize) -> Option<NetResult> {
if len < 8 {
return err(libc::EINVAL);
}
let mut bytes = [0u8; 8];
unsafe { std::ptr::copy_nonoverlapping(buf, bytes.as_mut_ptr(), 8) };
let add = u64::from_ne_bytes(bytes);
if add == u64::MAX {
return err(libc::EINVAL);
}
loop {
let mut socks = self.socks.lock().unwrap();
let Some(Sock::Event {
counter,
writes,
nonblocking,
ready_order,
ready_at,
..
}) = socks.get_mut(&fd)
else {
return err(libc::EINVAL);
};
if add <= (u64::MAX - 1).saturating_sub(*counter) {
*counter += add;
*writes += 1;
*ready_order = readiness_sequence();
*ready_at = self.shared().tstamp_now().mono;
break;
}
if *nonblocking {
return err(libc::EAGAIN);
}
drop(socks);
if !readiness().wait_until_dynamic(
"eventfd write",
None,
|keys| self.fd_interests([fd], keys),
|| add <= (u64::MAX - 1).saturating_sub(self.eventfd_peek(fd)),
) {
return err(libc::EAGAIN);
}
}
self.bump_fds([fd]);
ok(8)
}
}
#[cfg(target_os = "macos")]
mod bpf {
pub(super) const BIOCSETIF: u64 = 0x8020_426c;
pub(super) const BIOCIMMEDIATE: u64 = 0x8004_4270;
pub(super) const BIOCGBLEN: u64 = 0x4004_4266;
pub(super) const BPF_HDRLEN: usize = 18;
pub(super) const BUF_LEN: u32 = 4096;
}
impl Fs for Fabric {
#[cfg(target_os = "macos")]
fn owns(&self, fd: c_int) -> bool {
matches!(self.socks.lock().unwrap().get(&fd), Some(Sock::Raw { .. }))
}
#[cfg(target_os = "macos")]
unsafe fn open(
&self,
path: *const std::ffi::c_char,
flags: c_int,
_mode: u32,
) -> Option<NetResult> {
if path.is_null() {
return None;
}
let name = unsafe { std::ffi::CStr::from_ptr(path) }.to_str().ok()?;
if !name.starts_with("/dev/bpf") {
return None; }
if !self.shared().sys.privileges().root {
return err(libc::EACCES);
}
let nonblocking = flags & libc::O_NONBLOCK != 0;
self.open_socket(SocketKind::Packet, 0, |rec| Sock::Raw {
ifindex: None,
rx: VecDeque::new(),
arrived: 0,
nonblocking,
rec,
})
}
#[cfg(target_os = "macos")]
unsafe fn ioctl(&self, fd: c_int, request: u64, arg: i64) -> Option<NetResult> {
if !matches!(self.socks.lock().unwrap().get(&fd), Some(Sock::Raw { .. })) {
return None;
}
match request {
bpf::BIOCIMMEDIATE => ok(0),
bpf::BIOCGBLEN => {
if arg == 0 {
return err(libc::EFAULT);
}
unsafe { (arg as *mut u32).write_unaligned(bpf::BUF_LEN) };
ok(0)
}
bpf::BIOCSETIF => {
let base = arg as *const u8;
if base.is_null() {
return err(libc::EFAULT);
}
let mut name = Vec::new();
for i in 0..libc::IFNAMSIZ {
let b = unsafe { *base.add(i) };
if b == 0 {
break;
}
name.push(b);
}
let Ok(name) = String::from_utf8(name) else {
return err(libc::EINVAL);
};
let Some(index) = self.shared().topo().index_of(&name) else {
return err(libc::ENXIO);
};
if let Some(Sock::Raw { ifindex, .. }) = self.socks.lock().unwrap().get_mut(&fd) {
*ifindex = Some(index);
}
ok(0)
}
_ => None,
}
}
#[cfg(target_os = "macos")]
unsafe fn read(&self, fd: c_int, buf: *mut u8, len: usize) -> Option<NetResult> {
let frame = if let Some(f) = self.raw_pop(fd) {
f
} else if self.is_nonblocking(fd) {
return would_block();
} else if readiness().wait_until_dynamic(
"bpf read",
self.recv_deadline(fd),
|keys| self.fd_interests([fd], keys),
|| self.raw_has(fd),
) {
self.raw_pop(fd)?
} else {
return err(libc::EAGAIN); };
let fl = frame.len() as u32;
let mut out = vec![0u8; bpf::BPF_HDRLEN + frame.len()];
out[8..12].copy_from_slice(&fl.to_ne_bytes());
out[12..16].copy_from_slice(&fl.to_ne_bytes());
out[16..18].copy_from_slice(&(bpf::BPF_HDRLEN as u16).to_ne_bytes());
out[bpf::BPF_HDRLEN..].copy_from_slice(&frame);
let n = len.min(out.len());
unsafe { std::ptr::copy_nonoverlapping(out.as_ptr(), buf, n) };
ok(n as i64)
}
#[cfg(target_os = "macos")]
unsafe fn write(&self, fd: c_int, buf: *const u8, len: usize) -> Option<NetResult> {
self.raw_write(fd, buf, len)
}
#[cfg(target_os = "macos")]
unsafe fn close(&self, fd: c_int) -> Option<NetResult> {
unsafe { Net::close(self, fd) }
}
#[cfg(target_os = "macos")]
unsafe fn fd_replaced(&self, fd: c_int) -> Option<NetResult> {
unsafe { Net::fd_replaced(self, fd) }
}
}
#[cfg(test)]
mod socket_arena_tests {
use super::*;
fn event(counter: u64) -> Sock {
Sock::Event {
ready_order: 0,
ready_at: Duration::ZERO,
counter,
nonblocking: false,
semaphore: false,
writes: 0,
reads: 0,
}
}
fn counter(sockets: &Sockets, fd: c_int) -> u64 {
let Sock::Event { counter, .. } = sockets.get(&fd).unwrap() else {
panic!("expected event")
};
*counter
}
#[test]
fn aliases_share_status_flags_and_socket_state() {
crate::Sim::new().run(|| {
let mut sockets = Sockets::default();
sockets.insert(30, event(1));
sockets.alias(30, 10);
let id = sockets.identity(30).unwrap();
sockets.description_mut(&10).unwrap().status_flags |= libc::O_NONBLOCK;
let Sock::Event {
counter: value,
nonblocking,
..
} = sockets.get_mut(&10).unwrap()
else {
unreachable!()
};
*value = 7;
*nonblocking = true;
assert_eq!(counter(&sockets, 30), 7);
assert_eq!(sockets.identity(10), Some(id));
assert_eq!(
sockets.description_mut(&30).unwrap().status_flags,
libc::O_RDWR | libc::O_NONBLOCK
);
assert!(matches!(
sockets.get(&30),
Some(Sock::Event {
nonblocking: true,
..
})
));
assert!(sockets.remove(&10).unwrap().is_none());
assert_eq!(counter(&sockets, 30), 7);
assert_eq!(sockets.identity(30), Some(id));
assert!(sockets.remove(&30).unwrap().is_some());
});
}
#[test]
fn replacement_through_an_alias_keeps_identity_and_description_flags() {
crate::Sim::new().run(|| {
let mut sockets = Sockets::default();
sockets.insert(10, event(1));
sockets.alias(10, 20);
let id = sockets.identity(10).unwrap();
sockets.description_mut(&20).unwrap().status_flags |= libc::O_NONBLOCK;
sockets.insert(20, event(9));
assert_eq!(counter(&sockets, 10), 9);
assert_eq!(counter(&sockets, 20), 9);
assert_eq!(sockets.identity(10), Some(id));
assert_eq!(sockets.identity(20), Some(id));
assert_eq!(sockets.descriptions.len(), 1);
assert!(sockets.free_slots.is_empty());
assert_eq!(
sockets.description_mut(&10).unwrap().status_flags,
libc::O_RDWR | libc::O_NONBLOCK
);
});
}
#[test]
fn retired_slots_are_recycled_without_reusing_open_description_identity() {
crate::Sim::new().run(|| {
let mut sockets = Sockets::default();
sockets.insert(10, event(1));
sockets.alias(10, 20);
let previous = sockets.descriptors[&10];
assert!(sockets.remove(&10).unwrap().is_none());
assert!(sockets.free_slots.is_empty());
assert!(sockets.remove(&20).unwrap().is_some());
assert!(sockets.get(&10).is_none());
sockets.insert(10, event(2));
let current = sockets.descriptors[&10];
assert_eq!(current.slot, previous.slot);
assert_ne!(current.id, previous.id);
assert_eq!(sockets.descriptions.len(), 1);
assert_eq!(counter(&sockets, 10), 2);
#[cfg(target_os = "linux")]
{
assert!(sockets.get_identity(previous.id).is_none());
assert!(matches!(
sockets.get_identity(current.id),
Some(Sock::Event { counter: 2, .. })
));
}
sockets.insert(30, event(3));
let replaced = sockets.identity(30).unwrap();
assert!(sockets.remove(&30).unwrap().is_some());
sockets.alias(10, 30);
assert_eq!(sockets.identity(30), Some(current.id));
assert_eq!(counter(&sockets, 30), 2);
#[cfg(target_os = "linux")]
assert!(sockets.get_identity(replaced).is_none());
#[cfg(not(target_os = "linux"))]
let _ = replaced;
});
}
#[test]
fn description_iteration_uses_one_smallest_live_descriptor_per_alias_group() {
crate::Sim::new().run(|| {
let mut sockets = Sockets::default();
sockets.insert(30, event(1));
sockets.alias(30, 10);
sockets.alias(30, 20);
sockets.insert(40, event(2));
let mut descriptors: Vec<_> = sockets.iter().map(|(fd, _)| *fd).collect();
descriptors.sort();
assert_eq!(descriptors, [10, 20, 30, 40]);
let mut representatives: Vec<_> = sockets.iter_mut().map(|(fd, _)| *fd).collect();
representatives.sort();
assert_eq!(representatives, [10, 40]);
assert!(sockets.remove(&10).unwrap().is_none());
representatives = sockets.iter_mut().map(|(fd, _)| *fd).collect();
representatives.sort();
assert_eq!(representatives, [20, 40]);
assert_eq!(sockets.values().count(), 2);
});
}
}
#[cfg(test)]
mod payload_tests {
use super::*;
#[test]
fn inline_and_heap_payloads_preserve_bytes_and_owned_conversion() {
crate::Sim::new().run(|| {
println!(
"Datagram={} Payload={} Vec={} alignments={}/{}",
std::mem::size_of::<Datagram>(),
std::mem::size_of::<Payload>(),
std::mem::size_of::<Vec<u8>>(),
std::mem::align_of::<Payload>(),
std::mem::align_of::<Vec<u8>>()
);
for len in [0, 127, 128, 129, 65_507] {
let mut bytes: Vec<_> = (0..len).map(|i| (i % 251) as u8).collect();
let expected = bytes.clone();
let payload = Payload::copy_from(&bytes);
bytes.fill(255);
assert_eq!(&*payload, expected);
assert_eq!(&*payload.clone(), expected);
assert_eq!(matches!(payload, Payload::Inline { .. }), len <= 128);
let ptr = payload.as_ptr();
let converted = payload.into_vec();
assert_eq!(converted, expected);
if len > 128 {
assert_eq!(converted.as_ptr(), ptr);
}
}
});
}
}
#[cfg(test)]
mod udp_port_tests {
use super::*;
#[test]
fn endpoint_replacement_and_removal_preserve_station_membership() {
use std::os::fd::AsRawFd;
crate::Sim::new().run(|| {
let socket = std::net::UdpSocket::bind("127.0.0.1:9000").unwrap();
let rec = current()
.shared
.sockets
.lookup_fd(socket.as_raw_fd())
.unwrap();
let ordinary = Arc::new(DgramQueue {
rec: Some(rec),
..DgramQueue::default()
});
let endpoint = Arc::new(DgramQueue::default());
let mut registry = UdpRegistry::default();
let first: SocketAddr = "127.0.0.5:9400".parse().unwrap();
let second: SocketAddr = "127.0.0.5:9401".parse().unwrap();
registry.insert(first, endpoint.clone());
registry.insert(second, endpoint.clone());
registry.insert(first, ordinary);
assert!(registry.endpoints.contains_key(&first.ip()));
registry.remove(&second).unwrap();
assert!(!registry.endpoints.contains_key(&first.ip()));
registry.insert(first, endpoint.clone());
registry.insert(first, endpoint);
assert!(registry.endpoints.contains_key(&first.ip()));
registry.remove(&first).unwrap();
assert!(!registry.endpoints.contains_key(&first.ip()));
assert!(registry.candidates(first, false).is_empty());
});
}
#[test]
fn port_candidates_follow_bindings_through_bucket_removals() {
crate::Sim::new().run(|| {
let mut registry = UdpRegistry::default();
let addresses: Vec<SocketAddr> = [
"127.0.0.1:9400",
"0.0.0.0:9400",
"127.0.0.2:9400",
"[::1]:9400",
"127.0.0.1:9401",
]
.map(|addr| addr.parse().unwrap())
.to_vec();
for &address in &addresses {
registry.insert(address, Arc::new(DgramQueue::default()));
}
for &address in &addresses {
let replacement = Arc::new(DgramQueue::default());
registry.insert(address, replacement.clone());
assert!(
registry
.candidates(address, false)
.iter()
.find(|candidate| candidate.addr == address)
.is_some_and(|candidate| Arc::ptr_eq(&candidate.q, &replacement))
);
}
for &removed in &addresses {
for &destination in &addresses {
let mut actual: Vec<_> = registry
.candidates(destination, false)
.into_iter()
.map(|candidate| {
assert!(Arc::ptr_eq(&candidate.q, ®istry.bound[&candidate.addr]));
candidate.addr
})
.collect();
let mut expected: Vec<_> = registry
.bound
.keys()
.filter(|addr| addr.port() == destination.port())
.copied()
.collect();
actual.sort();
expected.sort();
assert_eq!(actual, expected);
}
registry.remove(&removed).unwrap();
}
assert!(registry.by_port.is_empty());
});
}
#[test]
fn allocation_wraps_and_reuses_both_edges_of_a_full_range() {
crate::Sim::new().run(|| {
let mut ports = UdpPorts::default();
ports.insert(UdpPorts::FIRST);
assert_eq!(ports.alloc_from(UdpPorts::FIRST), UdpPorts::FIRST + 1);
assert_eq!(ports.alloc_from(u16::MAX), u16::MAX);
ports.remove(UdpPorts::FIRST);
for port in UdpPorts::FIRST..=u16::MAX {
ports.insert(port);
}
assert_eq!(ports.alloc_from(UdpPorts::FIRST), 0);
assert_eq!(ports.alloc_from(u16::MAX), 0);
ports.remove(UdpPorts::FIRST);
assert_eq!(ports.alloc_from(u16::MAX), UdpPorts::FIRST);
ports.remove(u16::MAX);
assert_eq!(ports.alloc_from(u16::MAX - 1), u16::MAX);
assert_eq!(ports.alloc_from(UdpPorts::FIRST), UdpPorts::FIRST);
ports.insert(UdpPorts::FIRST);
assert_eq!(ports.alloc_from(UdpPorts::FIRST), u16::MAX);
ports.insert(u16::MAX);
assert_eq!(ports.alloc_from(UdpPorts::FIRST), 0);
});
}
#[test]
fn scoped_addresses_and_queue_replacement_keep_ports_occupied() {
crate::Sim::new().run(|| {
let mut registry = UdpRegistry::default();
let ip: Ipv6Addr = "fe80::1".parse().unwrap();
let first = SocketAddr::V6(std::net::SocketAddrV6::new(ip, 49152, 0, 1));
let second = SocketAddr::V6(std::net::SocketAddrV6::new(ip, 49152, 0, 2));
let queue = Arc::new(DgramQueue::default());
registry.insert(first, queue.clone());
registry.insert(first, Arc::new(DgramQueue::default()));
registry.insert(second, queue);
registry.remove(&first).unwrap();
assert_eq!(registry.alloc_port(IpAddr::V6(ip)), 49153);
assert_eq!(
registry.alloc_port(IpAddr::V6(Ipv6Addr::UNSPECIFIED)),
49153
);
registry.remove(&second).unwrap();
assert_eq!(registry.alloc_port(IpAddr::V6(ip)), 49152);
assert_eq!(
registry.alloc_port(IpAddr::V6(Ipv6Addr::UNSPECIFIED)),
49152
);
});
}
}