netforge 0.1.5

High-performance raw packet I/O engine with AF_XDP, sendmmsg, and pnet backends
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
//! AF_XDP zero-copy backend.
//!
//! AF_XDP (eXpress Data Path) bypasses the kernel network stack
//! entirely: packets are DMA-mapped into a userspace ring and the
//! application reads/writes them directly. Theoretical ceiling on a
//! modern NIC is 25–50M pps per queue; with 4 queues bound to 4 cores
//! you can saturate a 100GbE link with raw-socket-class scanning.
//!
//! # Implementation
//!
//! - Single [`xsk_rs::Umem`] of [`FRAME_COUNT`] frames.
//! - One [`xsk_rs::Socket`] bound to the requested interface, queue 0.
//! - Frames are split 50/50 between TX (free pool) and RX (pre-loaded
//!   into the fill queue at startup).
//! - `tx_batch`: drain the completion queue to reclaim sent frames,
//!   copy payload bytes into freshly-popped frames, submit via
//!   `produce_and_wakeup`.
//! - `rx_batch`: poll the RX queue, parse IP+TCP headers, copy
//!   payload, then return frames to the fill queue so the kernel can
//!   refill them with the next wave of arrivals.
//!
//! # Requirements
//!
//! - Linux 5.10+
//! - `CAP_BPF` (or root)
//! - libbpf-dev installed at link time
//! - An explicit interface name in [`crate::EngineConfig`] (no default)
//!
//! Construction returns [`EngineError::PermissionDenied`] on any
//! kernel-side failure so [`crate::engine::auto_select`] falls through
//! to sendmmsg/pnet on machines that cannot use XDP.

#[cfg(all(target_os = "linux", feature = "xdp-backend"))]
mod inner {
    use std::io::Write;
    use std::net::Ipv4Addr;
    use std::sync::Mutex;

    use xsk_rs::{
        CompQueue, FillQueue, FrameDesc, RxQueue, Socket, TxQueue, Umem,
        config::{QueueSize, SocketConfig, UmemConfig},
    };

    use crate::engine::{rate_limiter_from_pps, EngineConfig, EngineError, EngineStats, PacketEngine, RxPacket};
    use crate::packet::RawPacket;
    use crate::stats_window::PpsTracker;

    /// Total number of UMEM frames. Half are reserved for TX, half feed
    /// the fill queue for RX. 4096 × 4 KiB = 16 MiB UMEM footprint.
    const FRAME_COUNT: u32 = 4096;
    const TX_FRAMES: usize = (FRAME_COUNT as usize) / 2;
    /// Maximum packets handed to the kernel per TX submission. Larger
    /// batches amortise the wake-up syscall but increase tail latency.
    const MAX_TX_BATCH: usize = 256;
    /// Poll timeout for `poll_and_consume`. Zero = non-blocking; the
    /// caller of `rx_batch` is expected to drive the loop themselves.
    const RX_POLL_MS: i32 = 0;

    /// AF_XDP packet engine  -  zero-copy via XDP socket (XSK).
    ///
    /// Field order is load-bearing: Rust drops struct fields in declaration
    /// order, and the four queues (`tx_q`/`rx_q`/`fill_q`/`comp_q`) are built by
    /// `Socket::new(&umem, ...)` and hold resources backed by the UMEM mmap.
    /// `umem` is therefore declared LAST so it is dropped AFTER every queue; if
    /// it dropped first the queues would tear down against a freed mmap
    /// (drop-order use-after-free).
    pub struct XdpEngine {
        tx_q: Mutex<TxQueue>,
        rx_q: Mutex<RxQueue>,
        fill_q: Mutex<FillQueue>,
        comp_q: Mutex<CompQueue>,
        /// Free TX descriptors. Refilled from `comp_q` on each
        /// `tx_batch` call.
        tx_pool: Mutex<Vec<FrameDesc>>,
        tx_stats: PpsTracker,
        rx_stats: PpsTracker,
        #[allow(dead_code)]
        config: EngineConfig,
        rate_limiter: Option<ratelimit::TokenBucket>,
        /// Owns the UMEM mmap. Declared last so it outlives (drops after) the
        /// queues above, which reference its memory.
        umem: Umem,
    }

    /// Convert `EngineConfig` ring sizes into `xsk-rs` `QueueSize`s.
    ///
    /// A size of `0` falls back to the default `4096`; non-power-of-two
    /// sizes are rejected with `EngineError::InvalidConfig` because XDP
    /// rings must be powers of two.
    fn configured_queue_size(size: u32) -> Result<QueueSize, EngineError> {
        const DEFAULT: u32 = 4096;
        let size = if size == 0 { DEFAULT } else { size };
        QueueSize::new(size).map_err(|_| {
            EngineError::InvalidConfig(format!(
                "xdp ring size {size} is not a power of two (required by AF_XDP)"
            ))
        })
    }

    impl XdpEngine {
        /// Try to construct an XSK on the configured interface.
        ///
        /// # Errors
        ///
        /// Returns `EngineError::PermissionDenied` if the kernel
        /// rejects the bind (no `CAP_BPF`, interface doesn't exist,
        /// queue index out of range, etc.). `auto_select()` will then
        /// fall through to the sendmmsg backend.
        pub fn new(config: EngineConfig) -> Result<Self, EngineError> {
            let interface = config.interface.as_ref().ok_or_else(|| {
                EngineError::PermissionDenied(
                    "xdp-backend requires an explicit interface name".into(),
                )
            })?;

            let if_name = interface.parse().map_err(|e| {
                EngineError::PermissionDenied(format!("invalid interface name: {e:?}"))
            })?;

            let frame_count = std::num::NonZeroU32::new(FRAME_COUNT).ok_or_else(|| {
                EngineError::PermissionDenied("FRAME_COUNT must be > 0".into())
            })?;
            let tx_qsize = configured_queue_size(config.tx_ring_size)?;
            let rx_qsize = configured_queue_size(config.rx_ring_size)?;

            let umem_config = UmemConfig::builder()
                .fill_queue_size(rx_qsize)
                .comp_queue_size(tx_qsize)
                .build()
                .map_err(|e| EngineError::InvalidConfig(format!("xdp UMEM config invalid: {e}")))?;
            let (umem, mut descs) = Umem::new(umem_config, frame_count, false)
                .map_err(|e| {
                    EngineError::PermissionDenied(format!("xdp UMEM init failed: {e}"))
                })?;

            let socket_config = SocketConfig::builder()
                .tx_queue_size(tx_qsize)
                .rx_queue_size(rx_qsize)
                .build();

            // SAFETY: `Socket::new` is unsafe per xsk-rs docs. Caller
            // must ensure the kernel side (XDP program load, queue id
            // validity) lines up. We surface any kernel rejection as
            // PermissionDenied so auto_select falls through.
            let (tx_q, rx_q, fq_and_cq) = unsafe {
                Socket::new(socket_config, &umem, &if_name, 0)
            }
            .map_err(|e| {
                EngineError::PermissionDenied(format!("xdp socket bind failed: {e}"))
            })?;

            let (mut fill_q, comp_q) = fq_and_cq.ok_or_else(|| {
                EngineError::PermissionDenied(
                    "xdp socket did not yield a fill/completion queue pair".into(),
                )
            })?;

            // Split descs: tail half feeds TX free pool; head half is
            // immediately handed to the kernel via fill_q so RX has
            // landing slots from the very first poll.
            let tx_pool: Vec<FrameDesc> = descs.split_off(TX_FRAMES);
            // SAFETY: descs are valid UMEM frames freshly minted above;
            // the kernel will write packet bytes into them on RX and
            // hand the descriptors back via rx_q.
            unsafe {
                fill_q.produce(&descs);
            }

            tracing::info!(
                interface = %interface,
                frames = FRAME_COUNT,
                tx_pool = tx_pool.len(),
                rx_pool = descs.len(),
                "xdp engine bound successfully"
            );

            Ok(Self {
                umem,
                tx_q: Mutex::new(tx_q),
                rx_q: Mutex::new(rx_q),
                fill_q: Mutex::new(fill_q),
                comp_q: Mutex::new(comp_q),
                tx_pool: Mutex::new(tx_pool),
                tx_stats: PpsTracker::new(),
                rx_stats: PpsTracker::new(),
                rate_limiter: rate_limiter_from_pps(config.rate_pps),
                config,
            })
        }

        fn drain_completion(&self) {
            let mut comp_q = match self.comp_q.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };
            let mut pool = match self.tx_pool.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };
            // Reuse a per-thread scratch buffer to avoid per-call
            // allocation when the comp queue is busy. MAX_TX_BATCH is a
            // power of two so a fully-saturated TX cycle empties the comp
            // queue in one consume call. (Previously this allocated a
            // fresh Vec every call despite the "reuse" claim.)
            thread_local! {
                static COMP_SCRATCH: std::cell::RefCell<Vec<FrameDesc>> =
                    const { std::cell::RefCell::new(Vec::new()) };
            }
            COMP_SCRATCH.with_borrow_mut(|scratch| {
                if scratch.len() < MAX_TX_BATCH {
                    scratch.resize(MAX_TX_BATCH, FrameDesc::default());
                }
                // SAFETY: scratch holds MAX_TX_BATCH valid slots; consume
                // writes the returned descriptor count and overwrites only
                // the first `n` slots (which we then read).
                let consumed = unsafe { comp_q.consume(&mut scratch[..MAX_TX_BATCH]) };
                for desc in scratch.iter().take(consumed) {
                    pool.push(*desc);
                }
            });
        }
    }

    impl PacketEngine for XdpEngine {
        fn tx_batch(&self, packets: &[RawPacket]) -> Result<usize, EngineError> {
            if packets.is_empty() {
                return Ok(0);
            }

            // Reclaim previously-sent frames before allocating new ones
            //  -  a long-running TX loop drains the comp queue here on
            // every batch so the pool stays bounded.
            self.drain_completion();

            let mut pool = match self.tx_pool.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };
            let mut tx_q = match self.tx_q.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };

            let mut total_sent = 0usize;
            let mut cursor = 0usize;
            while cursor < packets.len() {
                // Limit the chunk by available tokens and available frames.
                let chunk_end = (cursor + MAX_TX_BATCH).min(packets.len());
                let mut chunk = &packets[cursor..chunk_end];

                if let Some(ref bucket) = self.rate_limiter {
                    let mut tokens = 0usize;
                    for _ in 0..chunk.len() {
                        if bucket.try_acquire() {
                            tokens += 1;
                        } else {
                            break;
                        }
                    }
                    if tokens < chunk.len() {
                        // No more rate-limit tokens; drop the rest and
                        // send what we already claimed.
                        self.tx_stats.record_drop((chunk.len() - tokens) as u64);
                        chunk = &chunk[..tokens];
                        cursor = chunk_end; // advance past this logical chunk
                        if chunk.is_empty() {
                            break;
                        }
                    } else {
                        cursor = chunk_end;
                    }
                } else {
                    cursor = chunk_end;
                }

                if pool.len() < chunk.len() {
                    // Pool exhausted  -  record drops for the remainder
                    // and bail. Caller gets back the partial count and
                    // can retry on the next loop tick.
                    let dropped = chunk.len().saturating_sub(pool.len());
                    self.tx_stats.record_drop(dropped as u64);
                    let take = pool.len();
                    if take == 0 {
                        break;
                    }
                    let chunk = &chunk[..take];
                    let used = submit_chunk(&self.umem, &mut tx_q, &mut pool, chunk)?;
                    self.tx_stats.record(used as u64);
                    total_sent += used;
                    break;
                }

                let used = submit_chunk(&self.umem, &mut tx_q, &mut pool, chunk)?;
                self.tx_stats.record(used as u64);
                total_sent += used;

                // If we were rate-limited mid-chunk, no more packets will be sent.
                if chunk.len() < MAX_TX_BATCH && cursor < packets.len() {
                    break;
                }
            }

            Ok(total_sent)
        }

        fn rx_batch(&self, buf: &mut [RxPacket]) -> Result<usize, EngineError> {
            if buf.is_empty() {
                return Ok(0);
            }

            let mut rx_q = match self.rx_q.lock() {
                Ok(g) => g,
                Err(p) => p.into_inner(),
            };

            // poll_and_consume requires a descriptor scratch buffer it
            // fills with whatever the kernel handed us this tick. Reuse a
            // per-thread buffer instead of allocating one every RX tick.
            thread_local! {
                static RX_DESC_SCRATCH: std::cell::RefCell<Vec<FrameDesc>> =
                    const { std::cell::RefCell::new(Vec::new()) };
            }
            RX_DESC_SCRATCH.with_borrow_mut(|descs| {
                if descs.len() < buf.len() {
                    descs.resize(buf.len(), FrameDesc::default());
                }
                // SAFETY: descs[..buf.len()] is valid storage; poll_and_consume
                // fills up to that many slots and returns the count. We only
                // read the first `n` slots after the call.
                let n = unsafe { rx_q.poll_and_consume(&mut descs[..buf.len()], RX_POLL_MS) }
                    .map_err(|e| {
                        EngineError::Io(std::io::Error::other(format!("xdp rx poll: {e}")))
                    })?;

                let mut received = 0usize;
                for desc in descs.iter().take(n) {
                    // SAFETY: desc was just produced by the kernel into our
                    // UMEM; the data slice is live until we return it via
                    // fill_q below.
                    let data = unsafe { self.umem.data(desc) };
                    let bytes = data.contents();
                    if let Some(parsed) = parse_rx_packet(bytes) {
                        buf[received] = parsed;
                        received += 1;
                    }
                }

                self.rx_stats.record(received as u64);

                // Hand the descriptors back to the kernel so it can refill
                // them with the next wave of packets. If we don't do this
                // RX silently stalls after `RX_FRAMES` packets.
                if n > 0 {
                    let mut fill_q = match self.fill_q.lock() {
                        Ok(g) => g,
                        Err(p) => p.into_inner(),
                    };
                    // SAFETY: descs[..n] are descriptors the kernel just
                    // handed us via rx_q; producing them back to fill_q is
                    // the documented round-trip per the xsk-rs API.
                    unsafe {
                        fill_q.produce(&descs[..n]);
                    }
                }

                Ok(received)
            })
        }

        fn stats(&self) -> EngineStats {
            EngineStats {
                tx_packets: self.tx_stats.total(),
                rx_packets: self.rx_stats.total(),
                tx_drops: self.tx_stats.drops(),
                rx_drops: self.rx_stats.drops(),
                tx_pps: self.tx_stats.snapshot_pps(),
                rx_pps: self.rx_stats.snapshot_pps(),
            }
        }

        fn name(&self) -> &'static str {
            "xdp"
        }
    }

    /// Copy each packet into its assigned UMEM frame and submit the
    /// batch to the kernel. Returns the number of descriptors actually
    /// produced (xsk-rs may refuse to enqueue if the TX ring is full)
    /// AND the unsubmitted tail (`descs[produced..]`).
    ///
    /// The caller MUST return the tail to `tx_pool`: those frames were
    /// popped from the pool and written to but never handed to the kernel,
    /// so they will never appear in the COMP ring and `drain_completion`
    /// can never reclaim them. Dropping the tail here (as an earlier
    /// version did) permanently shrinks the pool on every partial submit,
    /// exhausting it and stalling all TX.
    /// Submit one TX chunk, reusing a per-thread frame scratch buffer.
    ///
    /// Both `tx_batch` call sites route through here so the thread-local scratch
    /// buffer and the stamp/submit/recycle sequence have exactly one owner. The
    /// old code allocated a fresh `Vec<FrameDesc>` (`pool.drain(..).collect()`)
    /// plus a `split_off` Vec on every batch; this reuses one buffer per thread.
    ///
    /// Precondition: `pool.len() >= chunk.len()`.
    fn submit_chunk(
        umem: &Umem,
        tx_q: &mut TxQueue,
        pool: &mut Vec<FrameDesc>,
        chunk: &[RawPacket],
    ) -> Result<usize, EngineError> {
        thread_local! {
            static TX_FRAME_SCRATCH: std::cell::RefCell<Vec<FrameDesc>> =
                const { std::cell::RefCell::new(Vec::new()) };
        }
        TX_FRAME_SCRATCH.with_borrow_mut(|scratch| {
            marshal_and_submit_with(scratch, pool, chunk.len(), |descs| {
                stamp_and_submit(umem, tx_q, chunk, descs)
            })
        })
    }

    /// Stamp each packet into its frame and submit the batch to the TX ring.
    ///
    /// Operates on a borrowed `&mut [FrameDesc]` (a slice of the caller's reused
    /// scratch buffer) instead of an owned `Vec`, and returns only the number of
    /// frames the kernel accepted. The refused tail is `descs[produced..]`; the
    /// caller recycles it from the same scratch buffer with no split-off Vec
    /// allocation.
    fn stamp_and_submit(
        umem: &Umem,
        tx_q: &mut TxQueue,
        packets: &[RawPacket],
        descs: &mut [FrameDesc],
    ) -> Result<usize, EngineError> {
        debug_assert_eq!(packets.len(), descs.len());

        for (pkt, desc) in packets.iter().zip(descs.iter_mut()) {
            // SAFETY: `desc` was popped from tx_pool  -  we own it
            // exclusively until handed back to tx_q below. Writing into
            // its frame is safe.
            let mut data = unsafe { umem.data_mut(desc) };
            data.cursor()
                .write_all(pkt.as_bytes())
                .map_err(EngineError::Io)?;
        }

        // SAFETY: descs are owned by us right now; produce_and_wakeup
        // hands ownership of the produced prefix to the kernel TX ring +
        // COMP ring round trip.
        let produced = unsafe { tx_q.produce_and_wakeup(&*descs) }
            .map_err(|e| EngineError::Io(std::io::Error::other(format!("xdp tx submit: {e}"))))?;

        Ok(produced.min(descs.len()))
    }

    /// Drain `count` frames from `pool` into the reused `scratch` buffer, submit
    /// them via `submit`, and recycle the frames the kernel refused back into
    /// `pool` - all without allocating a fresh frames `Vec` per batch.
    ///
    /// `submit` is injected so the pure marshalling/recycling logic (the part the
    /// hot TX loop repeats every batch) is unit-testable without a real XDP umem
    /// or TX ring. Returns the number of frames accepted by `submit`.
    ///
    /// Precondition: `pool.len() >= count`.
    fn marshal_and_submit_with<S>(
        scratch: &mut Vec<FrameDesc>,
        pool: &mut Vec<FrameDesc>,
        count: usize,
        submit: S,
    ) -> Result<usize, EngineError>
    where
        S: FnOnce(&mut [FrameDesc]) -> Result<usize, EngineError>,
    {
        // Reuse the scratch buffer's existing capacity: clear() keeps the
        // allocation, extend() from the drain refills it in place.
        scratch.clear();
        scratch.extend(pool.drain(..count));

        let produced = submit(scratch.as_mut_slice())?.min(scratch.len());

        // Recycle the refused tail from the SAME buffer - no split_off Vec.
        pool.extend(scratch.drain(produced..));
        Ok(produced)
    }

    /// Parse an Ethernet-stripped IP+TCP packet handed up from the XDP
    /// ring into an [`RxPacket`]. Returns `None` for malformed or
    /// non-IPv4/non-TCP frames.
    fn parse_rx_packet(bytes: &[u8]) -> Option<RxPacket> {
        // Minimum frame: 14 (eth) + 20 (ip) + 20 (tcp) = 54.
        if bytes.len() < 54 {
            return None;
        }
        // Skip 14-byte Ethernet header. (XDP delivers L2 frames; the
        // upstream sendmmsg path operates on L3 because IP_HDRINCL
        // strips the L2 layer for us.)
        let ethertype = u16::from_be_bytes([bytes[12], bytes[13]]);
        if ethertype != 0x0800 {
            return None; // non-IPv4
        }
        let ip = &bytes[14..];
        let version = ip[0] >> 4;
        if version != 4 {
            return None;
        }
        let ihl = (ip[0] & 0x0F) as usize * 4;
        if ihl < 20 || ip.len() < ihl + 20 {
            return None;
        }
        let ip_total_len = u16::from_be_bytes([ip[2], ip[3]]) as usize;
        if ip_total_len < ihl + 20 || ip_total_len < ihl || ip_total_len > ip.len() {
            return None;
        }
        if ip[9] != 6 {
            return None; // non-TCP
        }
        let src_ip = Ipv4Addr::new(ip[12], ip[13], ip[14], ip[15]);
        let ttl = ip[8];
        let tcp = &ip[ihl..];
        let src_port = u16::from_be_bytes([tcp[0], tcp[1]]);
        let dst_port = u16::from_be_bytes([tcp[2], tcp[3]]);
        let seq_num = u32::from_be_bytes([tcp[4], tcp[5], tcp[6], tcp[7]]);
        let ack_num = u32::from_be_bytes([tcp[8], tcp[9], tcp[10], tcp[11]]);
        let tcp_data_offset = ((tcp[12] >> 4) as usize) * 4;
        let tcp_flags = tcp[13];
        let window = u16::from_be_bytes([tcp[14], tcp[15]]);
        let payload_start = ihl + tcp_data_offset;
        if tcp_data_offset < 20 || payload_start > ip.len() {
            return None;
        }
        let payload = if ip.len() > payload_start {
            ip[payload_start..].to_vec()
        } else {
            Vec::new()
        };
        // An oversized IP frame (> MAX_PACKET_SIZE) makes `from_bytes` return
        // None. The old `unwrap_or_default()` substituted an EMPTY RawPacket while
        // still emitting a fully-populated RxPacket, so a corrupted packet with a
        // bodyless `packet` field flowed downstream (Law 10). Drop it instead: an
        // unrepresentable frame is not a received packet.
        let packet = RawPacket::from_bytes(ip)?;
        Some(RxPacket {
            packet,
            src_ip,
            src_port,
            dst_port,
            tcp_flags,
            ack_num,
            seq_num,
            ttl,
            window,
            payload,
        })
    }

    // SAFETY: All mutating queues are protected by Mutex, the Umem is
    // only accessed via shared `data` / `data_mut` calls inside locked
    // sections, and the tx_pool is itself a Mutex.
    unsafe impl Send for XdpEngine {}
    unsafe impl Sync for XdpEngine {}

    #[cfg(test)]
    mod tests {
        use super::*;

        #[test]
        fn marshal_and_submit_recycles_refused_frames_into_the_pool() {
            let mut scratch: Vec<FrameDesc> = Vec::new();
            let mut pool: Vec<FrameDesc> = vec![FrameDesc::default(); 10];

            // Kernel accepts only 3 of the 5 submitted frames.
            let used = marshal_and_submit_with(&mut scratch, &mut pool, 5, |descs| {
                assert_eq!(descs.len(), 5, "all drained frames are handed to submit");
                Ok(3)
            })
            .expect("submit ok");

            assert_eq!(used, 3);
            // 5 drained, 3 consumed by the kernel, 2 refused and recycled:
            // pool = 10 - 5 + 2 = 7 (only the accepted frames left the pool).
            assert_eq!(pool.len(), 7, "refused frames must be recycled, not leaked");
            // Only the REFUSED tail is drained back to the pool; the produced
            // head (the kernel-owned frames) stays in scratch until the next
            // batch's `clear()`. It must never be recycled - that would hand the
            // pool descriptors the kernel still owns.
            assert_eq!(
                scratch.len(),
                used,
                "produced frames stay in scratch (cleared next batch), never recycled"
            );
        }

        #[test]
        fn marshal_and_submit_reuses_the_scratch_buffer_without_reallocating() {
            let mut scratch: Vec<FrameDesc> = Vec::new();
            let mut pool: Vec<FrameDesc> = vec![FrameDesc::default(); 64];

            // First batch warms the scratch buffer to steady-state capacity.
            marshal_and_submit_with(&mut scratch, &mut pool, 8, |_| Ok(8)).unwrap();
            // Refill the pool for a second identical batch.
            pool.extend(std::iter::repeat_with(FrameDesc::default).take(8));

            // The scratch buffer retains its allocation across batches: clear +
            // extend reuse it in place. Capture the buffer's identity, run
            // another batch, and assert the allocation did not move (the old
            // `pool.drain(..).collect()` allocated a fresh Vec every batch).
            scratch.reserve(MAX_TX_BATCH);
            let cap_before = scratch.capacity();
            let ptr_before = scratch.as_ptr();

            marshal_and_submit_with(&mut scratch, &mut pool, 8, |_| Ok(8)).unwrap();

            assert_eq!(
                scratch.capacity(),
                cap_before,
                "scratch buffer must not reallocate across batches"
            );
            assert_eq!(
                scratch.as_ptr(),
                ptr_before,
                "scratch buffer must be reused in place, not freshly allocated"
            );
        }

        #[test]
        fn parse_rejects_short_frame() {
            assert!(parse_rx_packet(&[0u8; 10]).is_none());
        }

        #[test]
        fn parse_rejects_non_ipv4() {
            let mut frame = vec![0u8; 80];
            // Set ethertype to IPv6 (0x86dd)
            frame[12] = 0x86;
            frame[13] = 0xdd;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_non_tcp() {
            let mut frame = vec![0u8; 80];
            // Ethertype = IPv4
            frame[12] = 0x08;
            frame[13] = 0x00;
            // IP version=4, IHL=5
            frame[14] = 0x45;
            // IP protocol = UDP (17), not TCP
            frame[14 + 9] = 17;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_extracts_tcp_header_fields() {
            let mut frame = vec![0u8; 80];
            // Ethertype IPv4
            frame[12] = 0x08;
            frame[13] = 0x00;
            // IP: version=4, IHL=5, total_len=66 (80 - 14 eth header), ttl=64, proto=TCP, src=10.0.0.1
            frame[14] = 0x45;
            frame[14 + 2..14 + 4].copy_from_slice(&66u16.to_be_bytes());
            frame[14 + 8] = 64;
            frame[14 + 9] = 6;
            frame[14 + 12..14 + 16].copy_from_slice(&[10, 0, 0, 1]);
            // TCP at offset 14+20=34: src_port=49152, dst_port=80,
            // seq=0x12345678, data_offset=5, flags=SYN+ACK, window=64240
            let tcp_off = 34;
            frame[tcp_off..tcp_off + 2].copy_from_slice(&49152u16.to_be_bytes());
            frame[tcp_off + 2..tcp_off + 4].copy_from_slice(&80u16.to_be_bytes());
            frame[tcp_off + 4..tcp_off + 8].copy_from_slice(&0x1234_5678u32.to_be_bytes());
            frame[tcp_off + 12] = 0x50;
            frame[tcp_off + 13] = 0x12; // SYN+ACK
            frame[tcp_off + 14..tcp_off + 16].copy_from_slice(&64240u16.to_be_bytes());
            let parsed = parse_rx_packet(&frame).expect("valid TCP frame should parse");
            assert_eq!(parsed.src_ip, Ipv4Addr::new(10, 0, 0, 1));
            assert_eq!(parsed.src_port, 49152);
            assert_eq!(parsed.dst_port, 80);
            assert_eq!(parsed.seq_num, 0x1234_5678);
            assert_eq!(parsed.ttl, 64);
            assert_eq!(parsed.window, 64240);
            assert_eq!(parsed.tcp_flags, 0x12);
        }

        #[test]
        fn parse_discards_oversized_frame_instead_of_emitting_empty_packet() {
            use crate::packet::MAX_PACKET_SIZE;
            // A frame whose IP portion exceeds MAX_PACKET_SIZE makes
            // RawPacket::from_bytes return None. The old unwrap_or_default()
            // emitted an RxPacket with a bodyless `packet` field; the fix drops
            // the frame entirely. Headers are otherwise valid so parsing reaches
            // the from_bytes call rather than bailing on an earlier check.
            let ip_len = MAX_PACKET_SIZE + 1; // one byte past what RawPacket holds
            let mut frame = vec![0u8; 14 + ip_len];
            // Ethertype IPv4.
            frame[12] = 0x08;
            frame[13] = 0x00;
            // IP: version=4, IHL=5, proto=TCP.
            frame[14] = 0x45;
            frame[14 + 9] = 6;
            // TCP data offset = 5 words so payload parsing stays in bounds.
            frame[34 + 12] = 0x50;
            assert!(
                parse_rx_packet(&frame).is_none(),
                "an oversized frame must be dropped, not emitted with an empty packet"
            );
        }

        #[test]
        fn parse_rejects_malformed_ihl_below_20() {
            let mut frame = vec![0u8; 80];
            frame[12] = 0x08;
            frame[13] = 0x00;
            // IHL = 4 (16 bytes, invalid, minimum is 5 / 20 bytes)
            frame[14] = 0x44;
            frame[14 + 9] = 6;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_malformed_tcp_data_offset_below_20() {
            let mut frame = vec![0u8; 80];
            frame[12] = 0x08;
            frame[13] = 0x00;
            frame[14] = 0x45;
            frame[14 + 9] = 6;
            // TCP data offset = 1 word (4 bytes, below minimum 20 bytes)
            frame[34 + 12] = 0x10;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_tcp_data_offset_past_ip_len() {
            let mut frame = vec![0u8; 80];
            frame[12] = 0x08;
            frame[13] = 0x00;
            frame[14] = 0x45;
            frame[14 + 9] = 6;
            // TCP data offset = 15 words (60 bytes), ihl=20 -> payload_start = 80, which is > ip.len() (66 bytes)
            frame[34 + 12] = 0xF0;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_non_ipv4_version_header() {
            let mut frame = vec![0u8; 80];
            frame[12] = 0x08;
            frame[13] = 0x00;
            frame[14] = 0x65; // Version 6, IHL 5
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_ip_total_len_below_header_plus_tcp() {
            let mut frame = vec![0u8; 80];
            frame[12] = 0x08;
            frame[13] = 0x00;
            frame[14] = 0x45;
            frame[14 + 2..14 + 4].copy_from_slice(&39u16.to_be_bytes()); // total len 39 < 40
            frame[14 + 9] = 6;
            assert!(parse_rx_packet(&frame).is_none());
        }

        #[test]
        fn parse_rejects_truncated_ip_total_len() {
            let mut frame = vec![0u8; 80];
            frame[12] = 0x08;
            frame[13] = 0x00;
            frame[14] = 0x45;
            frame[14 + 2..14 + 4].copy_from_slice(&100u16.to_be_bytes()); // total len 100 > captured 66
            frame[14 + 9] = 6;
            assert!(parse_rx_packet(&frame).is_none());
        }
    }
}

#[cfg(all(target_os = "linux", feature = "xdp-backend"))]
pub use inner::XdpEngine;