use std::os::fd::{AsFd, BorrowedFd, OwnedFd};
use crate::block::{Block, Frame, FrameMeta, PacketType};
use crate::dummy::{RxQueue, SentQueue, drain_readiness};
use crate::error::{Error, Result};
use crate::interface::{MacAddr, index_of};
use crate::sys::Stats;
use crate::sys::linux::ring::{RxRing, TxRing};
use crate::sys::linux::socket::{ETH_P_ALL, PacketSocket, SocketKind, poll_readable, poll_writable};
use crate::sys::linux::tpacket::{RingLayout, min_frame_size, tpacket_align};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Mode {
Ring,
Basic,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum FanoutMode {
Hash,
LoadBalance,
Cpu,
RollOver,
Random,
QueueMap,
}
impl FanoutMode {
fn raw(self) -> u16 {
let v = match self {
FanoutMode::Hash => libc::PACKET_FANOUT_HASH,
FanoutMode::LoadBalance => libc::PACKET_FANOUT_LB,
FanoutMode::Cpu => libc::PACKET_FANOUT_CPU,
FanoutMode::RollOver => libc::PACKET_FANOUT_ROLLOVER,
FanoutMode::Random => libc::PACKET_FANOUT_RND,
FanoutMode::QueueMap => libc::PACKET_FANOUT_QM,
};
v as u16
}
}
#[derive(Debug, Clone, Copy)]
pub struct RingConfig {
pub block_size: u32,
pub block_count: u32,
pub frame_size: u32,
pub retire_blk_tov_ms: u32,
pub snaplen: Option<u32>,
}
impl RingConfig {
#[must_use]
pub const fn rx_default() -> Self {
RingConfig {
block_size: 1 << 20,
block_count: 8,
frame_size: 2048,
retire_blk_tov_ms: 60,
snaplen: None,
}
}
#[must_use]
pub const fn tx_default() -> Self {
RingConfig {
block_size: 1 << 20,
block_count: 4,
frame_size: 2048,
retire_blk_tov_ms: 0,
snaplen: None,
}
}
#[must_use]
pub const fn small() -> Self {
RingConfig {
block_size: 1 << 16,
block_count: 4,
frame_size: 2048,
retire_blk_tov_ms: 60,
snaplen: None,
}
}
#[must_use]
pub const fn snaplen(mut self, bytes: u32) -> Self {
self.snaplen = Some(bytes);
self
}
fn to_layout(self) -> Result<RingLayout> {
let frame_size = match self.snaplen {
Some(snap) => frame_size_for_snaplen(snap).ok_or_else(|| {
Error::invalid_config(format!("snaplen {snap} is too large for a ring frame"))
})?,
None => self.frame_size,
};
RingLayout::new(
self.block_size,
self.block_count,
frame_size,
self.retire_blk_tov_ms,
page_size(),
)
}
}
fn frame_size_for_snaplen(snaplen: u32) -> Option<u32> {
let overhead = tpacket_align(libc::TPACKET3_HDRLEN) as u32 + 32;
let needed = overhead.checked_add(snaplen)?.max(min_frame_size() as u32);
needed.checked_next_power_of_two().map(|v| v.max(128))
}
fn page_size() -> u32 {
let v = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
if v <= 0 { 4096 } else { v as u32 }
}
#[derive(Debug, Clone)]
pub struct ChannelBuilder {
interface: String,
protocol: u16,
kind: SocketKind,
mode: Mode,
promiscuous: bool,
rx: RingConfig,
tx: RingConfig,
fanout: FanoutMode,
fanout_group: Option<u16>,
}
impl ChannelBuilder {
fn new(interface: impl Into<String>) -> Self {
ChannelBuilder {
interface: interface.into(),
protocol: ETH_P_ALL,
kind: SocketKind::Raw,
mode: Mode::Ring,
promiscuous: false,
rx: RingConfig::rx_default(),
tx: RingConfig::tx_default(),
fanout: FanoutMode::Hash,
fanout_group: None,
}
}
#[must_use]
pub fn protocol(mut self, protocol: u16) -> Self {
self.protocol = protocol;
self
}
#[must_use]
pub fn dgram(mut self) -> Self {
self.kind = SocketKind::Dgram;
self
}
#[must_use]
pub fn basic(mut self) -> Self {
self.mode = Mode::Basic;
self
}
#[must_use]
pub fn promiscuous(mut self, on: bool) -> Self {
self.promiscuous = on;
self
}
#[must_use]
pub fn fanout(mut self, mode: FanoutMode) -> Self {
self.fanout = mode;
self
}
#[must_use]
pub fn fanout_group(mut self, id: u16) -> Self {
self.fanout_group = Some(id);
self
}
#[must_use]
pub fn rx_ring(mut self, config: RingConfig) -> Self {
self.rx = config;
self
}
#[must_use]
pub fn tx_ring(mut self, config: RingConfig) -> Self {
self.tx = config;
self
}
pub fn build_sync(&self) -> Result<(Sender, Receiver)> {
let parts = self.open()?;
Ok((Sender { inner: parts.tx }, Receiver::new(parts.rx)))
}
pub(crate) fn open(&self) -> Result<ChannelParts> {
let idx = index_of(&self.interface)?;
Ok(ChannelParts { rx: self.open_rx_backend(idx)?, tx: self.open_tx_backend(idx)? })
}
fn open_rx_backend(&self, idx: crate::IfIndex) -> Result<RxBackend> {
let sock = PacketSocket::open(idx, self.protocol, self.kind)?;
if self.promiscuous {
sock.set_promiscuous(true)?;
}
match self.mode {
Mode::Ring => Ok(RxBackend::Ring(RxRing::open(sock, self.rx.to_layout()?)?)),
Mode::Basic => Ok(RxBackend::Basic { sock, buf: vec![0u8; 65_536] }),
}
}
fn open_tx_backend(&self, idx: crate::IfIndex) -> Result<TxBackend> {
let sock = PacketSocket::open(idx, self.protocol, self.kind)?;
match self.mode {
Mode::Ring => Ok(TxBackend::Ring(TxRing::open(sock, self.tx.to_layout()?)?)),
Mode::Basic => Ok(TxBackend::Basic(sock)),
}
}
pub(crate) fn build_fanout_backends(&self, count: usize) -> Result<Vec<RxBackend>> {
if count == 0 {
return Err(Error::invalid_config("fanout receiver count must be at least 1"));
}
let idx = index_of(&self.interface)?;
let mode = self.fanout.raw();
let mut backends = Vec::with_capacity(count);
let first = self.open_rx_backend(idx)?;
let group = match self.fanout_group {
Some(id) => {
first.set_fanout(id, mode)?;
id
}
None => first.set_fanout_unique(mode, default_group_id())?,
};
backends.push(first);
for _ in 1..count {
let backend = self.open_rx_backend(idx)?;
backend.set_fanout(group, mode)?;
backends.push(backend);
}
Ok(backends)
}
pub fn build_fanout_rx(&self, count: usize) -> Result<Vec<Receiver>> {
Ok(self
.build_fanout_backends(count)?
.into_iter()
.map(Receiver::new)
.collect())
}
}
fn default_group_id() -> u16 {
use std::sync::atomic::{AtomicU16, Ordering};
static SEQ: AtomicU16 = AtomicU16::new(0);
let pid = unsafe { libc::getpid() } as u16;
pid.wrapping_add(SEQ.fetch_add(1, Ordering::Relaxed))
}
pub(crate) struct ChannelParts {
pub(crate) rx: RxBackend,
pub(crate) tx: TxBackend,
}
#[derive(Debug)]
pub(crate) enum RxBackend {
Ring(RxRing),
Basic { sock: PacketSocket, buf: Vec<u8> },
Dummy { queue: RxQueue, readiness: OwnedFd, last: Vec<u8> },
}
impl RxBackend {
#[cfg(feature = "tokio")]
pub(crate) fn borrow_fd(&self) -> BorrowedFd<'_> {
match self {
RxBackend::Ring(ring) => ring.as_fd(),
RxBackend::Basic { sock, .. } => sock.as_fd(),
RxBackend::Dummy { readiness, .. } => readiness.as_fd(),
}
}
pub(crate) fn set_fanout(&self, group_id: u16, mode: u16) -> Result<()> {
match self {
RxBackend::Ring(ring) => ring.set_fanout(group_id, mode),
RxBackend::Basic { sock, .. } => sock.set_fanout(group_id, mode),
RxBackend::Dummy { .. } => Ok(()),
}
}
pub(crate) fn set_fanout_unique(&self, mode: u16, fallback_id: u16) -> Result<u16> {
match self {
RxBackend::Ring(ring) => ring.set_fanout_unique(mode, fallback_id),
RxBackend::Basic { sock, .. } => sock.set_fanout_unique(mode, fallback_id),
RxBackend::Dummy { .. } => Ok(fallback_id),
}
}
}
#[derive(Debug)]
pub(crate) enum TxBackend {
Ring(TxRing),
Basic(PacketSocket),
Dummy { sent: SentQueue, fd: OwnedFd },
}
impl TxBackend {
#[cfg(feature = "tokio")]
pub(crate) fn borrow_fd(&self) -> BorrowedFd<'_> {
match self {
TxBackend::Ring(tx) => tx.as_fd(),
TxBackend::Basic(sock) => sock.as_fd(),
TxBackend::Dummy { fd, .. } => fd.as_fd(),
}
}
pub(crate) fn try_send(&mut self, frame: &[u8]) -> std::io::Result<usize> {
use crate::sys::RawChannel;
match self {
TxBackend::Ring(tx) => tx.send(frame),
TxBackend::Basic(sock) => sock.send(frame),
TxBackend::Dummy { sent, .. } => {
sent.lock().expect("dummy queue poisoned").push_back(frame.to_vec());
Ok(frame.len())
}
}
}
pub(crate) fn try_send_batch(&mut self, frames: &[&[u8]]) -> std::io::Result<usize> {
use crate::sys::RawChannel;
match self {
TxBackend::Ring(tx) => tx.send_batch(frames),
TxBackend::Basic(sock) => {
let mut sent = 0;
for frame in frames {
match sock.send(frame) {
Ok(_) => sent += 1,
Err(e) if sent > 0 && e.kind() == std::io::ErrorKind::WouldBlock => break,
Err(e) => return Err(e),
}
}
Ok(sent)
}
TxBackend::Dummy { sent, .. } => {
let mut q = sent.lock().expect("dummy queue poisoned");
for frame in frames {
q.push_back(frame.to_vec());
}
Ok(frames.len())
}
}
}
pub(crate) fn max_frame_len(&self) -> Option<usize> {
match self {
TxBackend::Ring(tx) => Some(tx.max_frame_len()),
TxBackend::Basic(_) | TxBackend::Dummy { .. } => None,
}
}
#[cfg(feature = "tokio")]
pub(crate) fn has_unsent(&self) -> bool {
match self {
TxBackend::Ring(tx) => tx.has_unsent(),
TxBackend::Basic(_) | TxBackend::Dummy { .. } => false,
}
}
pub(crate) fn finish_kick(&self) -> std::io::Result<()> {
match self {
TxBackend::Ring(tx) => tx.finish_kick(),
TxBackend::Basic(_) | TxBackend::Dummy { .. } => Ok(()),
}
}
}
#[derive(Debug)]
pub struct Channel;
impl Channel {
#[must_use]
pub fn builder(interface: impl Into<String>) -> ChannelBuilder {
ChannelBuilder::new(interface)
}
}
#[derive(Debug)]
pub struct Sender {
inner: TxBackend,
}
impl Sender {
pub(crate) fn from_dummy(sent: SentQueue, fd: OwnedFd) -> Self {
Sender { inner: TxBackend::Dummy { sent, fd } }
}
pub fn send(&mut self, frame: &[u8]) -> Result<()> {
if let Some(max) = self.inner.max_frame_len() {
if frame.len() > max {
return Err(Error::FrameTooLarge { len: frame.len(), max });
}
}
loop {
match self.inner.try_send(frame) {
Ok(_) => break,
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
poll_writable(self.as_fd(), 100).map_err(Error::Send)?;
self.inner.finish_kick().map_err(Error::Send)?;
}
Err(e) => return Err(Error::Send(e)),
}
}
self.inner.finish_kick().map_err(Error::Send)
}
pub fn send_batch(&mut self, frames: &[&[u8]]) -> Result<usize> {
if frames.is_empty() {
return Ok(0);
}
check_batch_frame_sizes(frames, self.inner.max_frame_len())?;
let sent = loop {
match self.inner.try_send_batch(frames) {
Ok(n) => break n,
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
poll_writable(self.as_fd(), 100).map_err(Error::Send)?;
self.inner.finish_kick().map_err(Error::Send)?;
}
Err(e) => return Err(Error::Send(e)),
}
};
self.inner.finish_kick().map_err(Error::Send)?;
Ok(sent)
}
pub fn as_fd(&self) -> BorrowedFd<'_> {
match &self.inner {
TxBackend::Ring(tx) => tx.as_fd(),
TxBackend::Basic(sock) => sock.as_fd(),
TxBackend::Dummy { fd, .. } => fd.as_fd(),
}
}
}
#[derive(Debug)]
pub struct Receiver {
inner: RxBackend,
stats_total: Stats,
}
impl Receiver {
pub(crate) fn new(inner: RxBackend) -> Self {
Receiver { inner, stats_total: Stats::default() }
}
pub(crate) fn from_dummy(queue: RxQueue, readiness: OwnedFd) -> Self {
Receiver::new(RxBackend::Dummy { queue, readiness, last: Vec::new() })
}
pub fn recv_block(&mut self) -> Result<Block<'_>> {
match &mut self.inner {
RxBackend::Ring(ring) => {
loop {
if ring.block_ready() {
break;
}
poll_readable(ring.as_fd(), -1).map_err(Error::Recv)?;
}
Ok(ring.recv_block().expect("block reported ready"))
}
RxBackend::Basic { sock, buf } => {
let (n, pkttype) = loop {
match sock.recv_into(buf) {
Ok(v) => break v,
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
poll_readable(sock.as_fd(), -1).map_err(Error::Recv)?;
}
Err(e) => return Err(Error::Recv(e)),
}
};
let captured = n.min(buf.len());
let meta = FrameMeta {
wire_len: n,
timestamp: None,
vlan: None,
packet_type: PacketType::from_raw(pkttype),
};
Ok(Block::single(Some(Frame::new(&buf[..captured], meta))))
}
RxBackend::Dummy { queue, readiness, last } => {
let frame = loop {
let popped = queue.lock().expect("dummy queue poisoned").pop_front();
match popped {
Some(Ok(v)) => break v,
Some(Err(e)) => return Err(Error::Recv(e)),
None => {
poll_readable(readiness.as_fd(), -1).map_err(Error::Recv)?;
let eof = drain_readiness(readiness.as_fd());
if eof && queue.lock().expect("dummy queue poisoned").is_empty() {
return Err(Error::Recv(std::io::Error::from(
std::io::ErrorKind::UnexpectedEof,
)));
}
}
}
};
*last = frame;
let meta = FrameMeta {
wire_len: last.len(),
timestamp: None,
vlan: None,
packet_type: dummy_packet_type(last),
};
Ok(Block::single(Some(Frame::new(&last[..], meta))))
}
}
}
pub fn stats(&mut self) -> Result<Stats> {
let delta = match &mut self.inner {
RxBackend::Ring(ring) => ring.stats().map_err(Error::Stats)?,
RxBackend::Basic { sock, .. } => sock.statistics().map_err(Error::Stats)?,
RxBackend::Dummy { .. } => Stats::default(),
};
self.stats_total.accumulate(delta);
Ok(self.stats_total)
}
pub fn set_promiscuous(&self, on: bool) -> Result<()> {
match &self.inner {
RxBackend::Ring(ring) => ring.set_promiscuous(on),
RxBackend::Basic { sock, .. } => sock.set_promiscuous(on),
RxBackend::Dummy { .. } => Ok(()),
}
}
pub fn as_fd(&self) -> BorrowedFd<'_> {
match &self.inner {
RxBackend::Ring(ring) => ring.as_fd(),
RxBackend::Basic { sock, .. } => sock.as_fd(),
RxBackend::Dummy { readiness, .. } => readiness.as_fd(),
}
}
}
pub(crate) fn check_batch_frame_sizes(frames: &[&[u8]], max: Option<usize>) -> Result<()> {
if let Some(max) = max {
if let Some(f) = frames.iter().find(|f| f.len() > max) {
return Err(Error::FrameTooLarge { len: f.len(), max });
}
}
Ok(())
}
pub(crate) fn dummy_packet_type(frame: &[u8]) -> PacketType {
frame
.get(0..6)
.map(|b| PacketType::from_dest_mac(MacAddr([b[0], b[1], b[2], b[3], b[4], b[5]])))
.unwrap_or(PacketType::Other(0))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn builder_defaults() {
let b = Channel::builder("eth0");
assert_eq!(b.protocol, ETH_P_ALL);
assert_eq!(b.mode, Mode::Ring);
assert!(!b.promiscuous);
}
#[test]
fn builder_is_chainable() {
let b = Channel::builder("eth0").dgram().basic().promiscuous(true).protocol(0x0800);
assert_eq!(b.mode, Mode::Basic);
assert!(b.promiscuous);
assert_eq!(b.protocol, 0x0800);
assert_eq!(b.kind, SocketKind::Dgram);
}
#[test]
#[cfg_attr(miri, ignore = "to_layout reads the page size via sysconf")]
fn ring_config_validates() {
assert!(RingConfig::rx_default().to_layout().is_ok());
assert!(RingConfig::tx_default().to_layout().is_ok());
let bad = RingConfig { block_size: 100, ..RingConfig::rx_default() };
assert!(bad.to_layout().is_err());
}
#[test]
fn fanout_zero_receivers_is_invalid_config() {
let r = Channel::builder("definitely-not-an-iface-xyz").build_fanout_rx(0);
assert!(matches!(r, Err(Error::InvalidConfig(_))));
}
#[test]
#[cfg_attr(miri, ignore = "calls if_nametoindex")]
fn build_on_missing_interface_errors() {
let r = Channel::builder("definitely-not-an-iface-xyz").build_sync();
assert!(matches!(r, Err(Error::InterfaceNotFound(_))));
}
#[test]
#[cfg_attr(miri, ignore = "to_layout reads the page size via sysconf")]
fn small_preset_is_compact_and_valid() {
let layout = RingConfig::small().to_layout().expect("small layout valid");
assert_eq!(layout.map_len(), (1usize << 16) * 4); }
#[test]
#[cfg_attr(miri, ignore = "to_layout reads the page size via sysconf")]
fn absurd_snaplen_is_invalid_config_not_panic() {
let r = RingConfig::rx_default().snaplen(u32::MAX).to_layout();
assert!(matches!(r, Err(Error::InvalidConfig(_))));
let msg = r.unwrap_err().to_string();
assert!(msg.contains(&u32::MAX.to_string()), "message should name the value: {msg}");
let r = RingConfig::rx_default().snaplen((1 << 31) - 1).to_layout();
assert!(matches!(r, Err(Error::InvalidConfig(_))));
}
#[test]
#[cfg_attr(miri, ignore = "to_layout reads the page size via sysconf")]
fn snaplen_derives_dense_power_of_two_frames() {
assert_eq!(frame_size_for_snaplen(1500), Some(2048)); assert_eq!(frame_size_for_snaplen(128), Some(256)); assert!(frame_size_for_snaplen(1).unwrap() >= 128);
assert!(frame_size_for_snaplen(64).unwrap().is_power_of_two());
let layout = RingConfig::small().snaplen(128).to_layout().expect("snaplen layout valid");
assert_eq!(layout.frame_size, 256);
assert_eq!(layout.frame_count, layout.map_len() as u32 / 256);
}
proptest::proptest! {
#[test]
#[cfg_attr(miri, ignore = "proptest is slow under Miri and covers safe arithmetic")]
fn snaplen_frame_size_invariants(snaplen in 0u32..=65535) {
let fs = frame_size_for_snaplen(snaplen).expect("sane snaplen always fits");
proptest::prop_assert!(fs.is_power_of_two());
proptest::prop_assert!(fs >= 128);
proptest::prop_assert!(fs as usize >= min_frame_size());
proptest::prop_assert_eq!((1u32 << 20) % fs, 0);
}
}
}