#[cfg(all(target_os = "linux", feature = "xdp-backend"))]
mod inner {
use std::io::Write;
use std::net::Ipv4Addr;
use std::sync::Mutex;
use xsk_rs::{
CompQueue, FillQueue, FrameDesc, RxQueue, Socket, TxQueue, Umem,
config::{SocketConfig, UmemConfig},
};
use crate::engine::{EngineConfig, EngineError, EngineStats, PacketEngine, RxPacket};
use crate::packet::RawPacket;
use crate::stats_window::PpsTracker;
const FRAME_COUNT: u32 = 4096;
const TX_FRAMES: usize = (FRAME_COUNT as usize) / 2;
const MAX_TX_BATCH: usize = 256;
const RX_POLL_MS: i32 = 0;
pub struct XdpEngine {
umem: Umem,
tx_q: Mutex<TxQueue>,
rx_q: Mutex<RxQueue>,
fill_q: Mutex<FillQueue>,
comp_q: Mutex<CompQueue>,
tx_pool: Mutex<Vec<FrameDesc>>,
tx_stats: PpsTracker,
rx_stats: PpsTracker,
#[allow(dead_code)]
config: EngineConfig,
}
impl XdpEngine {
pub fn new(config: EngineConfig) -> Result<Self, EngineError> {
let interface = config.interface.as_ref().ok_or_else(|| {
EngineError::PermissionDenied(
"xdp-backend requires an explicit interface name".into(),
)
})?;
let if_name = interface.parse().map_err(|e| {
EngineError::PermissionDenied(format!("invalid interface name: {e:?}"))
})?;
let frame_count = std::num::NonZeroU32::new(FRAME_COUNT).ok_or_else(|| {
EngineError::PermissionDenied("FRAME_COUNT must be > 0".into())
})?;
let (umem, mut descs) = Umem::new(UmemConfig::default(), frame_count, false)
.map_err(|e| {
EngineError::PermissionDenied(format!("xdp UMEM init failed: {e}"))
})?;
let (tx_q, rx_q, fq_and_cq) = unsafe {
Socket::new(SocketConfig::default(), &umem, &if_name, 0)
}
.map_err(|e| {
EngineError::PermissionDenied(format!("xdp socket bind failed: {e}"))
})?;
let (mut fill_q, comp_q) = fq_and_cq.ok_or_else(|| {
EngineError::PermissionDenied(
"xdp socket did not yield a fill/completion queue pair".into(),
)
})?;
let tx_pool: Vec<FrameDesc> = descs.split_off(TX_FRAMES);
unsafe {
fill_q.produce(&descs);
}
tracing::info!(
interface = %interface,
frames = FRAME_COUNT,
tx_pool = tx_pool.len(),
rx_pool = descs.len(),
"xdp engine bound successfully"
);
Ok(Self {
umem,
tx_q: Mutex::new(tx_q),
rx_q: Mutex::new(rx_q),
fill_q: Mutex::new(fill_q),
comp_q: Mutex::new(comp_q),
tx_pool: Mutex::new(tx_pool),
tx_stats: PpsTracker::new(),
rx_stats: PpsTracker::new(),
config,
})
}
fn drain_completion(&self) {
let mut comp_q = match self.comp_q.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
let mut pool = match self.tx_pool.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
let mut scratch: Vec<FrameDesc> = vec![FrameDesc::default(); MAX_TX_BATCH];
let consumed = unsafe { comp_q.consume(&mut scratch) };
for desc in scratch.into_iter().take(consumed) {
pool.push(desc);
}
}
}
impl PacketEngine for XdpEngine {
fn tx_batch(&self, packets: &[RawPacket]) -> Result<usize, EngineError> {
if packets.is_empty() {
return Ok(0);
}
self.drain_completion();
let mut pool = match self.tx_pool.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
let mut tx_q = match self.tx_q.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
let mut total_sent = 0usize;
let mut cursor = 0usize;
while cursor < packets.len() {
let chunk_end = (cursor + MAX_TX_BATCH).min(packets.len());
let chunk = &packets[cursor..chunk_end];
cursor = chunk_end;
if pool.len() < chunk.len() {
let dropped = chunk.len().saturating_sub(pool.len());
self.tx_stats.record_drop(dropped as u64);
let take = pool.len();
if take == 0 {
break;
}
let chunk = &chunk[..take];
let used: Vec<FrameDesc> = pool.drain(..take).collect();
let used = stamp_and_submit(&self.umem, &mut tx_q, chunk, used)?;
self.tx_stats.record(used as u64);
total_sent += used;
break;
}
let used: Vec<FrameDesc> = pool.drain(..chunk.len()).collect();
let used = stamp_and_submit(&self.umem, &mut tx_q, chunk, used)?;
self.tx_stats.record(used as u64);
total_sent += used;
}
Ok(total_sent)
}
fn rx_batch(&self, buf: &mut [RxPacket]) -> Result<usize, EngineError> {
if buf.is_empty() {
return Ok(0);
}
let mut rx_q = match self.rx_q.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
let mut descs: Vec<FrameDesc> = vec![FrameDesc::default(); buf.len()];
let n = unsafe { rx_q.poll_and_consume(&mut descs, RX_POLL_MS) }
.map_err(|e| EngineError::Io(std::io::Error::other(format!("xdp rx poll: {e}"))))?;
let mut received = 0usize;
for desc in descs.iter().take(n) {
let data = unsafe { self.umem.data(desc) };
let bytes = data.contents();
if let Some(parsed) = parse_rx_packet(bytes) {
buf[received] = parsed;
received += 1;
}
}
self.rx_stats.record(received as u64);
if n > 0 {
let mut fill_q = match self.fill_q.lock() {
Ok(g) => g,
Err(p) => p.into_inner(),
};
unsafe {
fill_q.produce(&descs[..n]);
}
}
Ok(received)
}
fn stats(&self) -> EngineStats {
EngineStats {
tx_packets: self.tx_stats.total(),
rx_packets: self.rx_stats.total(),
tx_drops: self.tx_stats.drops(),
rx_drops: self.rx_stats.drops(),
tx_pps: self.tx_stats.snapshot_pps(),
rx_pps: self.rx_stats.snapshot_pps(),
}
}
fn name(&self) -> &'static str {
"xdp"
}
}
fn stamp_and_submit(
umem: &Umem,
tx_q: &mut TxQueue,
packets: &[RawPacket],
mut descs: Vec<FrameDesc>,
) -> Result<usize, EngineError> {
debug_assert_eq!(packets.len(), descs.len());
for (pkt, desc) in packets.iter().zip(descs.iter_mut()) {
let mut data = unsafe { umem.data_mut(desc) };
data.cursor()
.write_all(pkt.as_bytes())
.map_err(EngineError::Io)?;
}
let produced = unsafe { tx_q.produce_and_wakeup(&descs) }
.map_err(|e| EngineError::Io(std::io::Error::other(format!("xdp tx submit: {e}"))))?;
Ok(produced)
}
fn parse_rx_packet(bytes: &[u8]) -> Option<RxPacket> {
if bytes.len() < 54 {
return None;
}
let ethertype = u16::from_be_bytes([bytes[12], bytes[13]]);
if ethertype != 0x0800 {
return None; }
let ip = &bytes[14..];
let ihl = (ip[0] & 0x0F) as usize * 4;
if ip.len() < ihl + 20 {
return None;
}
if ip[9] != 6 {
return None; }
let src_ip = Ipv4Addr::new(ip[12], ip[13], ip[14], ip[15]);
let ttl = ip[8];
let tcp = &ip[ihl..];
let src_port = u16::from_be_bytes([tcp[0], tcp[1]]);
let dst_port = u16::from_be_bytes([tcp[2], tcp[3]]);
let seq_num = u32::from_be_bytes([tcp[4], tcp[5], tcp[6], tcp[7]]);
let ack_num = u32::from_be_bytes([tcp[8], tcp[9], tcp[10], tcp[11]]);
let tcp_data_offset = ((tcp[12] >> 4) as usize) * 4;
let tcp_flags = tcp[13];
let window = u16::from_be_bytes([tcp[14], tcp[15]]);
let payload_start = ihl + tcp_data_offset;
let payload = if ip.len() > payload_start {
ip[payload_start..].to_vec()
} else {
Vec::new()
};
Some(RxPacket {
packet: RawPacket::from_bytes(ip).unwrap_or_default(),
src_ip,
src_port,
dst_port,
tcp_flags,
ack_num,
seq_num,
ttl,
window,
payload,
})
}
unsafe impl Send for XdpEngine {}
unsafe impl Sync for XdpEngine {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_rejects_short_frame() {
assert!(parse_rx_packet(&[0u8; 10]).is_none());
}
#[test]
fn parse_rejects_non_ipv4() {
let mut frame = vec![0u8; 80];
frame[12] = 0x86;
frame[13] = 0xdd;
assert!(parse_rx_packet(&frame).is_none());
}
#[test]
fn parse_rejects_non_tcp() {
let mut frame = vec![0u8; 80];
frame[12] = 0x08;
frame[13] = 0x00;
frame[14] = 0x45;
frame[14 + 9] = 17;
assert!(parse_rx_packet(&frame).is_none());
}
#[test]
fn parse_extracts_tcp_header_fields() {
let mut frame = vec![0u8; 80];
frame[12] = 0x08;
frame[13] = 0x00;
frame[14] = 0x45;
frame[14 + 8] = 64;
frame[14 + 9] = 6;
frame[14 + 12..14 + 16].copy_from_slice(&[10, 0, 0, 1]);
let tcp_off = 34;
frame[tcp_off..tcp_off + 2].copy_from_slice(&49152u16.to_be_bytes());
frame[tcp_off + 2..tcp_off + 4].copy_from_slice(&80u16.to_be_bytes());
frame[tcp_off + 4..tcp_off + 8].copy_from_slice(&0x1234_5678u32.to_be_bytes());
frame[tcp_off + 12] = 0x50;
frame[tcp_off + 13] = 0x12; frame[tcp_off + 14..tcp_off + 16].copy_from_slice(&64240u16.to_be_bytes());
let parsed = parse_rx_packet(&frame).expect("valid TCP frame should parse");
assert_eq!(parsed.src_ip, Ipv4Addr::new(10, 0, 0, 1));
assert_eq!(parsed.src_port, 49152);
assert_eq!(parsed.dst_port, 80);
assert_eq!(parsed.seq_num, 0x1234_5678);
assert_eq!(parsed.ttl, 64);
assert_eq!(parsed.window, 64240);
assert_eq!(parsed.tcp_flags, 0x12);
}
}
}
#[cfg(all(target_os = "linux", feature = "xdp-backend"))]
pub use inner::XdpEngine;