netforge 0.1.0

High-performance raw packet I/O engine with AF_XDP, sendmmsg, and pnet backends
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
//! AF_XDP zero-copy backend.
//!
//! AF_XDP (eXpress Data Path) bypasses the kernel network stack
//! entirely: packets are DMA-mapped into a userspace ring and the
//! application reads/writes them directly. Theoretical ceiling on a
//! modern NIC is 25–50M pps per queue; with 4 queues bound to 4 cores
//! you can saturate a 100GbE link with raw-socket-class scanning.
//!
//! # Implementation
//!
//! - Single [`xsk_rs::Umem`] of [`FRAME_COUNT`] frames.
//! - One [`xsk_rs::Socket`] bound to the requested interface, queue 0.
//! - Frames are split 50/50 between TX (free pool) and RX (pre-loaded
//!   into the fill queue at startup).
//! - `tx_batch`: drain the completion queue to reclaim sent frames,
//!   copy payload bytes into freshly-popped frames, submit via
//!   `produce_and_wakeup`.
//! - `rx_batch`: poll the RX queue, parse IP+TCP headers, copy
//!   payload, then return frames to the fill queue so the kernel can
//!   refill them with the next wave of arrivals.
//!
//! # Requirements
//!
//! - Linux 5.10+
//! - `CAP_BPF` (or root)
//! - libbpf-dev installed at link time
//! - An explicit interface name in [`crate::EngineConfig`] (no default)
//!
//! Construction returns [`EngineError::PermissionDenied`] on any
//! kernel-side failure so [`crate::engine::auto_select`] falls through
//! to sendmmsg/pnet on machines that cannot use XDP.

#[cfg(all(target_os = "linux", feature = "xdp-backend"))]
mod inner {
    use std::io::Write;
    use std::net::Ipv4Addr;
    use std::sync::Mutex;

    use xsk_rs::{
        CompQueue, FillQueue, FrameDesc, RxQueue, Socket, TxQueue, Umem,
        config::{SocketConfig, UmemConfig},
    };

    use crate::engine::{EngineConfig, EngineError, EngineStats, PacketEngine, RxPacket};
    use crate::packet::RawPacket;
    use crate::stats_window::PpsTracker;

    /// Total number of UMEM frames. Half are reserved for TX, half feed
    /// the fill queue for RX. 4096 × 4 KiB = 16 MiB UMEM footprint.
    const FRAME_COUNT: u32 = 4096;
    const TX_FRAMES: usize = (FRAME_COUNT as usize) / 2;
    /// Maximum packets handed to the kernel per TX submission. Larger
    /// batches amortise the wake-up syscall but increase tail latency.
    const MAX_TX_BATCH: usize = 256;
    /// Poll timeout for `poll_and_consume`. Zero = non-blocking; the
    /// caller of `rx_batch` is expected to drive the loop themselves.
    const RX_POLL_MS: i32 = 0;

    /// AF_XDP packet engine — zero-copy via XDP socket (XSK).
    pub struct XdpEngine {
        umem: Umem,
        tx_q: Mutex<TxQueue>,
        rx_q: Mutex<RxQueue>,
        fill_q: Mutex<FillQueue>,
        comp_q: Mutex<CompQueue>,
        /// Free TX descriptors. Refilled from `comp_q` on each
        /// `tx_batch` call.
        tx_pool: Mutex<Vec<FrameDesc>>,
        tx_stats: PpsTracker,
        rx_stats: PpsTracker,
        #[allow(dead_code)]
        config: EngineConfig,
    }

    impl XdpEngine {
        /// Try to construct an XSK on the configured interface.
        ///
        /// # Errors
        ///
        /// Returns `EngineError::PermissionDenied` if the kernel
        /// rejects the bind (no `CAP_BPF`, interface doesn't exist,
        /// queue index out of range, etc.). `auto_select()` will then
        /// fall through to the sendmmsg backend.
        pub fn new(config: EngineConfig) -> Result<Self, EngineError> {
            let interface = config.interface.as_ref().ok_or_else(|| {
                EngineError::PermissionDenied(
                    "xdp-backend requires an explicit interface name".into(),
                )
            })?;

            let if_name = interface.parse().map_err(|e| {
                EngineError::PermissionDenied(format!("invalid interface name: {e:?}"))
            })?;

            let frame_count = std::num::NonZeroU32::new(FRAME_COUNT).ok_or_else(|| {
                EngineError::PermissionDenied("FRAME_COUNT must be > 0".into())
            })?;
            let (umem, mut descs) = Umem::new(UmemConfig::default(), frame_count, false)
                .map_err(|e| {
                    EngineError::PermissionDenied(format!("xdp UMEM init failed: {e}"))
                })?;

            // SAFETY: `Socket::new` is unsafe per xsk-rs docs. Caller
            // must ensure the kernel side (XDP program load, queue id
            // validity) lines up. We surface any kernel rejection as
            // PermissionDenied so auto_select falls through.
            let (tx_q, rx_q, fq_and_cq) = unsafe {
                Socket::new(SocketConfig::default(), &umem, &if_name, 0)
            }
            .map_err(|e| {
                EngineError::PermissionDenied(format!("xdp socket bind failed: {e}"))
            })?;

            let (mut fill_q, comp_q) = fq_and_cq.ok_or_else(|| {
                EngineError::PermissionDenied(
                    "xdp socket did not yield a fill/completion queue pair".into(),
                )
            })?;

            // Split descs: tail half feeds TX free pool; head half is
            // immediately handed to the kernel via fill_q so RX has
            // landing slots from the very first poll.
            let tx_pool: Vec<FrameDesc> = descs.split_off(TX_FRAMES);
            // SAFETY: descs are valid UMEM frames freshly minted above;
            // the kernel will write packet bytes into them on RX and
            // hand the descriptors back via rx_q.
            unsafe {
                fill_q.produce(&descs);
            }

            tracing::info!(
                interface = %interface,
                frames = FRAME_COUNT,
                tx_pool = tx_pool.len(),
                rx_pool = descs.len(),
                "xdp engine bound successfully"
            );

            Ok(Self {
                umem,
                tx_q: Mutex::new(tx_q),
                rx_q: Mutex::new(rx_q),
                fill_q: Mutex::new(fill_q),
                comp_q: Mutex::new(comp_q),
                tx_pool: Mutex::new(tx_pool),
                tx_stats: PpsTracker::new(),
                rx_stats: PpsTracker::new(),
                config,
            })
        }

        fn drain_completion(&self) {
            let mut comp_q = match self.comp_q.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };
            let mut pool = match self.tx_pool.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };
            // Reuse a small scratch buffer to avoid per-call allocation
            // when the comp queue is busy. 256 is a power-of-two that
            // matches MAX_TX_BATCH so a fully-saturated TX cycle
            // empties the comp queue in one consume call.
            let mut scratch: Vec<FrameDesc> = vec![FrameDesc::default(); MAX_TX_BATCH];
            // SAFETY: scratch is valid storage; consume writes the
            // returned descriptor count and overwrites only the first
            // `n` slots.
            let consumed = unsafe { comp_q.consume(&mut scratch) };
            for desc in scratch.into_iter().take(consumed) {
                pool.push(desc);
            }
        }
    }

    impl PacketEngine for XdpEngine {
        fn tx_batch(&self, packets: &[RawPacket]) -> Result<usize, EngineError> {
            if packets.is_empty() {
                return Ok(0);
            }

            // Reclaim previously-sent frames before allocating new ones
            // — a long-running TX loop drains the comp queue here on
            // every batch so the pool stays bounded.
            self.drain_completion();

            let mut pool = match self.tx_pool.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };
            let mut tx_q = match self.tx_q.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };

            let mut total_sent = 0usize;
            let mut cursor = 0usize;
            while cursor < packets.len() {
                let chunk_end = (cursor + MAX_TX_BATCH).min(packets.len());
                let chunk = &packets[cursor..chunk_end];
                cursor = chunk_end;

                if pool.len() < chunk.len() {
                    // Pool exhausted — record drops for the remainder
                    // and bail. Caller gets back the partial count and
                    // can retry on the next loop tick.
                    let dropped = chunk.len().saturating_sub(pool.len());
                    self.tx_stats.record_drop(dropped as u64);
                    let take = pool.len();
                    if take == 0 {
                        break;
                    }
                    let chunk = &chunk[..take];
                    let used: Vec<FrameDesc> = pool.drain(..take).collect();
                    let used = stamp_and_submit(&self.umem, &mut tx_q, chunk, used)?;
                    self.tx_stats.record(used as u64);
                    total_sent += used;
                    break;
                }

                let used: Vec<FrameDesc> = pool.drain(..chunk.len()).collect();
                let used = stamp_and_submit(&self.umem, &mut tx_q, chunk, used)?;
                self.tx_stats.record(used as u64);
                total_sent += used;
            }

            Ok(total_sent)
        }

        fn rx_batch(&self, buf: &mut [RxPacket]) -> Result<usize, EngineError> {
            if buf.is_empty() {
                return Ok(0);
            }

            let mut rx_q = match self.rx_q.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };

            // poll_and_consume requires a descriptor scratch buffer it
            // fills with whatever the kernel handed us this tick.
            let mut descs: Vec<FrameDesc> = vec![FrameDesc::default(); buf.len()];
            // SAFETY: descs is valid storage; poll_and_consume fills
            // up to `descs.len()` slots and returns the count. We only
            // read the first `n` slots after the call.
            let n = unsafe { rx_q.poll_and_consume(&mut descs, RX_POLL_MS) }
                .map_err(|e| EngineError::Io(std::io::Error::other(format!("xdp rx poll: {e}"))))?;

            let mut received = 0usize;
            for desc in descs.iter().take(n) {
                // SAFETY: desc was just produced by the kernel into our
                // UMEM; the data slice is live until we return it via
                // fill_q below.
                let data = unsafe { self.umem.data(desc) };
                let bytes = data.contents();
                if let Some(parsed) = parse_rx_packet(bytes) {
                    buf[received] = parsed;
                    received += 1;
                }
            }

            self.rx_stats.record(received as u64);

            // Hand the descriptors back to the kernel so it can refill
            // them with the next wave of packets. If we don't do this
            // RX silently stalls after `RX_FRAMES` packets.
            if n > 0 {
                let mut fill_q = match self.fill_q.lock() {
                    Ok(g) => g,
                    Err(p) => p.into_inner(),
                };
                // SAFETY: descs[..n] are descriptors the kernel just
                // handed us via rx_q; producing them back to fill_q is
                // the documented round-trip per the xsk-rs API.
                unsafe {
                    fill_q.produce(&descs[..n]);
                }
            }

            Ok(received)
        }

        fn stats(&self) -> EngineStats {
            EngineStats {
                tx_packets: self.tx_stats.total(),
                rx_packets: self.rx_stats.total(),
                tx_drops: self.tx_stats.drops(),
                rx_drops: self.rx_stats.drops(),
                tx_pps: self.tx_stats.snapshot_pps(),
                rx_pps: self.rx_stats.snapshot_pps(),
            }
        }

        fn name(&self) -> &'static str {
            "xdp"
        }
    }

    /// Copy each packet into its assigned UMEM frame and submit the
    /// batch to the kernel. Returns the number of descriptors actually
    /// produced (xsk-rs may refuse to enqueue if the TX ring is full).
    fn stamp_and_submit(
        umem: &Umem,
        tx_q: &mut TxQueue,
        packets: &[RawPacket],
        mut descs: Vec<FrameDesc>,
    ) -> Result<usize, EngineError> {
        debug_assert_eq!(packets.len(), descs.len());

        for (pkt, desc) in packets.iter().zip(descs.iter_mut()) {
            // SAFETY: `desc` was popped from tx_pool — we own it
            // exclusively until handed back to tx_q below. Writing into
            // its frame is safe.
            let mut data = unsafe { umem.data_mut(desc) };
            data.cursor()
                .write_all(pkt.as_bytes())
                .map_err(EngineError::Io)?;
        }

        // SAFETY: descs are owned by us right now; produce_and_wakeup
        // hands ownership to the kernel TX ring + COMP ring round
        // trip.
        let produced = unsafe { tx_q.produce_and_wakeup(&descs) }
            .map_err(|e| EngineError::Io(std::io::Error::other(format!("xdp tx submit: {e}"))))?;

        Ok(produced)
    }

    /// Parse an Ethernet-stripped IP+TCP packet handed up from the XDP
    /// ring into an [`RxPacket`]. Returns `None` for malformed or
    /// non-IPv4/non-TCP frames.
    fn parse_rx_packet(bytes: &[u8]) -> Option<RxPacket> {
        // Minimum frame: 14 (eth) + 20 (ip) + 20 (tcp) = 54.
        if bytes.len() < 54 {
            return None;
        }
        // Skip 14-byte Ethernet header. (XDP delivers L2 frames; the
        // upstream sendmmsg path operates on L3 because IP_HDRINCL
        // strips the L2 layer for us.)
        let ethertype = u16::from_be_bytes([bytes[12], bytes[13]]);
        if ethertype != 0x0800 {
            return None; // non-IPv4
        }
        let ip = &bytes[14..];
        let ihl = (ip[0] & 0x0F) as usize * 4;
        if ip.len() < ihl + 20 {
            return None;
        }
        if ip[9] != 6 {
            return None; // non-TCP
        }
        let src_ip = Ipv4Addr::new(ip[12], ip[13], ip[14], ip[15]);
        let ttl = ip[8];
        let tcp = &ip[ihl..];
        let src_port = u16::from_be_bytes([tcp[0], tcp[1]]);
        let dst_port = u16::from_be_bytes([tcp[2], tcp[3]]);
        let seq_num = u32::from_be_bytes([tcp[4], tcp[5], tcp[6], tcp[7]]);
        let ack_num = u32::from_be_bytes([tcp[8], tcp[9], tcp[10], tcp[11]]);
        let tcp_data_offset = ((tcp[12] >> 4) as usize) * 4;
        let tcp_flags = tcp[13];
        let window = u16::from_be_bytes([tcp[14], tcp[15]]);
        let payload_start = ihl + tcp_data_offset;
        let payload = if ip.len() > payload_start {
            ip[payload_start..].to_vec()
        } else {
            Vec::new()
        };
        Some(RxPacket {
            packet: RawPacket::from_bytes(ip).unwrap_or_default(),
            src_ip,
            src_port,
            dst_port,
            tcp_flags,
            ack_num,
            seq_num,
            ttl,
            window,
            payload,
        })
    }

    // SAFETY: All mutating queues are protected by Mutex, the Umem is
    // only accessed via shared `data` / `data_mut` calls inside locked
    // sections, and the tx_pool is itself a Mutex.
    unsafe impl Send for XdpEngine {}
    unsafe impl Sync for XdpEngine {}

    #[cfg(test)]
    mod tests {
        use super::*;

        #[test]
        fn parse_rejects_short_frame() {
            assert!(parse_rx_packet(&[0u8; 10]).is_none());
        }

        #[test]
        fn parse_rejects_non_ipv4() {
            let mut frame = vec![0u8; 80];
            // Set ethertype to IPv6 (0x86dd)
            frame[12] = 0x86;
            frame[13] = 0xdd;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_non_tcp() {
            let mut frame = vec![0u8; 80];
            // Ethertype = IPv4
            frame[12] = 0x08;
            frame[13] = 0x00;
            // IP version=4, IHL=5
            frame[14] = 0x45;
            // IP protocol = UDP (17), not TCP
            frame[14 + 9] = 17;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_extracts_tcp_header_fields() {
            let mut frame = vec![0u8; 80];
            // Ethertype IPv4
            frame[12] = 0x08;
            frame[13] = 0x00;
            // IP: version=4, IHL=5, ttl=64, proto=TCP, src=10.0.0.1
            frame[14] = 0x45;
            frame[14 + 8] = 64;
            frame[14 + 9] = 6;
            frame[14 + 12..14 + 16].copy_from_slice(&[10, 0, 0, 1]);
            // TCP at offset 14+20=34: src_port=49152, dst_port=80,
            // seq=0x12345678, data_offset=5, flags=SYN+ACK, window=64240
            let tcp_off = 34;
            frame[tcp_off..tcp_off + 2].copy_from_slice(&49152u16.to_be_bytes());
            frame[tcp_off + 2..tcp_off + 4].copy_from_slice(&80u16.to_be_bytes());
            frame[tcp_off + 4..tcp_off + 8].copy_from_slice(&0x1234_5678u32.to_be_bytes());
            frame[tcp_off + 12] = 0x50;
            frame[tcp_off + 13] = 0x12; // SYN+ACK
            frame[tcp_off + 14..tcp_off + 16].copy_from_slice(&64240u16.to_be_bytes());
            let parsed = parse_rx_packet(&frame).expect("valid TCP frame should parse");
            assert_eq!(parsed.src_ip, Ipv4Addr::new(10, 0, 0, 1));
            assert_eq!(parsed.src_port, 49152);
            assert_eq!(parsed.dst_port, 80);
            assert_eq!(parsed.seq_num, 0x1234_5678);
            assert_eq!(parsed.ttl, 64);
            assert_eq!(parsed.window, 64240);
            assert_eq!(parsed.tcp_flags, 0x12);
        }
    }
}

#[cfg(all(target_os = "linux", feature = "xdp-backend"))]
pub use inner::XdpEngine;