yip-io 0.1.0

High-performance I/O engines (AF_XDP zero-copy, io_uring, and vectorized recvmmsg/sendmmsg) for yip
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
//! Self-contained eBPF XSK redirect driver and map loader.
//!
//! Provides a zero-dependency Linux eBPF/XDP driver using direct `bpf(2)` syscalls
//! to direct incoming tunnel UDP datagrams straight into AF_XDP rings (`BPF_MAP_TYPE_XSKMAP`)
//! at the NIC driver layer, bypassing `sk_buff` allocations.
//!
//! All raw syscalls and pointer manipulations are quarantined within this module,
//! with explicit `// SAFETY:` justifications on every unsafe block.

use std::ffi::CString;
use std::io;
use std::os::fd::RawFd;

/// Operational status of the opportunistic eBPF XDP filter attachment.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BpfFilterStatus {
    /// Filter successfully loaded and attached to the network interface.
    Attached(RawFd),
    /// Unprivileged execution (missing `CAP_BPF` / `CAP_NET_ADMIN` / `EPERM`).
    FallbackUnprivileged,
    /// BPF or XDP unsupported on the interface or kernel.
    Unsupported,
}

// BPF syscall command constants
pub const BPF_MAP_CREATE: libc::c_int = 0;
pub const BPF_MAP_UPDATE_ELEM: libc::c_int = 2;
pub const BPF_PROG_LOAD: libc::c_int = 5;
pub const BPF_LINK_CREATE: libc::c_int = 28;

// BPF types
pub const BPF_MAP_TYPE_XSKMAP: u32 = 17;
pub const BPF_PROG_TYPE_XDP: u32 = 6;
pub const BPF_XDP: u32 = 37;

// BPF helper function IDs
pub const BPF_FUNC_REDIRECT_MAP: i32 = 51;

// XDP return actions
pub const XDP_PASS: i32 = 2;

// Pseudo map FD flag for BPF_LD_IMM64
pub const BPF_PSEUDO_MAP_FD: u8 = 1;

// eBPF instruction encoding constants
pub const BPF_LD: u8 = 0x00;
pub const BPF_LDX: u8 = 0x01;
pub const BPF_ALU64: u8 = 0x07;
pub const BPF_JMP: u8 = 0x05;

pub const BPF_W: u8 = 0x00;
pub const BPF_H: u8 = 0x08;
pub const BPF_B: u8 = 0x10;
pub const BPF_DW: u8 = 0x18;

pub const BPF_IMM: u8 = 0x00;
pub const BPF_MEM: u8 = 0x60;

pub const BPF_K: u8 = 0x00;
pub const BPF_X: u8 = 0x08;

pub const BPF_ADD: u8 = 0x00;
pub const BPF_AND: u8 = 0x50;
pub const BPF_LSH: u8 = 0x60;
pub const BPF_MOV: u8 = 0xb0;

pub const BPF_JA: u8 = 0x00;
pub const BPF_JEQ: u8 = 0x10;
pub const BPF_JGT: u8 = 0x20;
pub const BPF_JNE: u8 = 0x50;
pub const BPF_CALL: u8 = 0x80;
pub const BPF_EXIT: u8 = 0x90;

// Registers
pub const R0: u8 = 0;
pub const R1: u8 = 1;
pub const R2: u8 = 2;
pub const R3: u8 = 3;
pub const R4: u8 = 4;
pub const R5: u8 = 5;
pub const R6: u8 = 6;

/// 8-byte eBPF instruction binary layout matching Linux kernel ABI (`struct bpf_insn`).
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BpfInsn {
    pub code: u8,
    pub regs: u8, // (src_reg << 4) | (dst_reg & 0x0f)
    pub off: i16,
    pub imm: i32,
}

impl BpfInsn {
    pub const fn new(code: u8, dst: u8, src: u8, off: i16, imm: i32) -> Self {
        Self {
            code,
            regs: ((src & 0x0f) << 4) | (dst & 0x0f),
            off,
            imm,
        }
    }

    pub const fn mov64_reg(dst: u8, src: u8) -> Self {
        Self::new(BPF_ALU64 | BPF_MOV | BPF_X, dst, src, 0, 0)
    }

    pub const fn mov64_imm(dst: u8, imm: i32) -> Self {
        Self::new(BPF_ALU64 | BPF_MOV | BPF_K, dst, 0, 0, imm)
    }

    pub const fn add64_imm(dst: u8, imm: i32) -> Self {
        Self::new(BPF_ALU64 | BPF_ADD | BPF_K, dst, 0, 0, imm)
    }

    pub const fn add64_reg(dst: u8, src: u8) -> Self {
        Self::new(BPF_ALU64 | BPF_ADD | BPF_X, dst, src, 0, 0)
    }

    pub const fn and64_imm(dst: u8, imm: i32) -> Self {
        Self::new(BPF_ALU64 | BPF_AND | BPF_K, dst, 0, 0, imm)
    }

    pub const fn lsh64_imm(dst: u8, imm: i32) -> Self {
        Self::new(BPF_ALU64 | BPF_LSH | BPF_K, dst, 0, 0, imm)
    }

    pub const fn ja(off: i16) -> Self {
        Self::new(BPF_JMP | BPF_JA, 0, 0, off, 0)
    }

    pub const fn jeq_imm(dst: u8, imm: i32, off: i16) -> Self {
        Self::new(BPF_JMP | BPF_JEQ | BPF_K, dst, 0, off, imm)
    }

    pub const fn ldx_mem_w(dst: u8, src: u8, off: i16) -> Self {
        Self::new(BPF_LDX | BPF_MEM | BPF_W, dst, src, off, 0)
    }

    pub const fn ldx_mem_h(dst: u8, src: u8, off: i16) -> Self {
        Self::new(BPF_LDX | BPF_MEM | BPF_H, dst, src, off, 0)
    }

    pub const fn ldx_mem_b(dst: u8, src: u8, off: i16) -> Self {
        Self::new(BPF_LDX | BPF_MEM | BPF_B, dst, src, off, 0)
    }

    pub const fn jgt_reg(dst: u8, src: u8, off: i16) -> Self {
        Self::new(BPF_JMP | BPF_JGT | BPF_X, dst, src, off, 0)
    }

    pub const fn jne_imm(dst: u8, imm: i32, off: i16) -> Self {
        Self::new(BPF_JMP | BPF_JNE | BPF_K, dst, 0, off, imm)
    }

    pub const fn call_helper(func: i32) -> Self {
        Self::new(BPF_JMP | BPF_CALL, 0, 0, 0, func)
    }

    pub const fn exit() -> Self {
        Self::new(BPF_JMP | BPF_EXIT, 0, 0, 0, 0)
    }
}

#[repr(C)]
#[derive(Clone, Copy)]
pub struct BpfMapCreateAttr {
    pub map_type: u32,
    pub key_size: u32,
    pub value_size: u32,
    pub max_entries: u32,
    pub map_flags: u32,
}

#[repr(C)]
#[derive(Clone, Copy)]
pub struct BpfMapElemAttr {
    pub map_fd: u32,
    pub _pad: u32,
    pub key: u64,
    pub value: u64,
    pub flags: u64,
}

#[repr(C)]
#[derive(Clone, Copy)]
pub struct BpfProgLoadAttr {
    pub prog_type: u32,
    pub insn_cnt: u32,
    pub insns: u64,
    pub license: u64,
    pub log_level: u32,
    pub log_size: u32,
    pub log_buf: u64,
    pub kern_version: u32,
    pub prog_flags: u32,
}

#[repr(C)]
#[derive(Clone, Copy)]
pub struct BpfLinkCreateAttr {
    pub prog_fd: u32,
    pub target_ifindex: u32,
    pub attach_type: u32,
    pub flags: u32,
}

/// Linux `union bpf_attr` representation, 152 bytes.
#[repr(C)]
pub union BpfAttr {
    pub map_create: BpfMapCreateAttr,
    pub map_elem: BpfMapElemAttr,
    pub prog_load: BpfProgLoadAttr,
    pub link_create: BpfLinkCreateAttr,
    pub pad: [u8; 152],
}

/// Raw syscall wrapper for Linux `bpf(2)`.
///
/// # Safety
///
/// Caller must ensure `attr` points to an initialized memory buffer of at least `size`
/// bytes matching the kernel's ABI for command `cmd`, or NULL if the command expects no attributes.
pub unsafe fn sys_bpf(cmd: libc::c_int, attr: *const libc::c_void, size: usize) -> libc::c_long {
    // SAFETY: Invokes the Linux bpf syscall with caller-verified arguments.
    libc::syscall(libc::SYS_bpf, cmd, attr, size)
}

/// Creates a `BPF_MAP_TYPE_XSKMAP` map holding up to `max_entries` AF_XDP socket descriptors.
pub fn bpf_map_create_xsk(max_entries: u32) -> io::Result<RawFd> {
    let mut attr = BpfAttr { pad: [0u8; 152] };
    attr.map_create = BpfMapCreateAttr {
        map_type: BPF_MAP_TYPE_XSKMAP,
        key_size: 4,
        value_size: 4,
        max_entries,
        map_flags: 0,
    };

    // SAFETY: sys_bpf is called with BPF_MAP_CREATE, a valid pointer to stack-local
    // BpfAttr, and exact size of BpfAttr matching the kernel ABI.
    let res = unsafe {
        sys_bpf(
            BPF_MAP_CREATE,
            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
            std::mem::size_of::<BpfAttr>(),
        )
    };

    if res < 0 {
        return Err(io::Error::last_os_error());
    }
    Ok(res as RawFd)
}

/// Updates an entry in an XSK map mapping `queue_id` to `xsk_fd`.
pub fn bpf_map_update_xsk(map_fd: RawFd, queue_id: u32, xsk_fd: RawFd) -> io::Result<()> {
    let mut attr = BpfAttr { pad: [0u8; 152] };
    let key = queue_id;
    let value = xsk_fd as u32;

    attr.map_elem = BpfMapElemAttr {
        map_fd: map_fd as u32,
        _pad: 0,
        key: std::ptr::addr_of!(key) as u64,
        value: std::ptr::addr_of!(value) as u64,
        flags: 0,
    };

    // SAFETY: sys_bpf is called with BPF_MAP_UPDATE_ELEM, attr points to valid
    // initialized BpfMapElemAttr, key and value point to valid stack locals that
    // outlive the syscall, and size matches sizeof(BpfAttr).
    let res = unsafe {
        sys_bpf(
            BPF_MAP_UPDATE_ELEM,
            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
            std::mem::size_of::<BpfAttr>(),
        )
    };

    if res < 0 {
        return Err(io::Error::last_os_error());
    }
    Ok(())
}

/// Assembles the dual-stack outer UDP eBPF XDP filter bytecode.
///
/// Matches both IPv4 (`0x0800`) and IPv6 (`0x86dd`) outer UDP datagrams targeting `listen_port`,
/// steering matched packets directly to core-pinned AF_XDP rings via `bpf_redirect_map(xsk_map_fd, ctx->rx_queue_index, 0)`.
pub fn bpf_prog_assemble_xdp(listen_port: u16, xsk_map_fd: RawFd) -> Vec<BpfInsn> {
    let eth_p_ip = (0x0800u16).to_be() as i32;
    let eth_p_ipv6 = (0x86ddu16).to_be() as i32;
    let udp_port = listen_port.to_be() as i32;

    vec![
        // 0: r6 = r1 (save context pointer)
        BpfInsn::mov64_reg(R6, R1),
        // 1: r2 = *(u32 *)(r6 + 0) (ctx->data)
        BpfInsn::ldx_mem_w(R2, R6, 0),
        // 2: r3 = *(u32 *)(r6 + 4) (ctx->data_end)
        BpfInsn::ldx_mem_w(R3, R6, 4),
        // 3: r4 = r2
        BpfInsn::mov64_reg(R4, R2),
        // 4: r4 += 14 (Ethernet header length)
        BpfInsn::add64_imm(R4, 14),
        // 5: if r4 > r3 goto PASS (target index 39: off = 39 - (5 + 1) = 33)
        BpfInsn::jgt_reg(R4, R3, 33),
        // 6: r4 = *(u16 *)(r2 + 12) (eth->h_proto)
        BpfInsn::ldx_mem_h(R4, R2, 12),
        // 7: if r4 == eth_p_ip goto IPV4 (target index 17: off = 17 - (7 + 1) = 9)
        BpfInsn::jeq_imm(R4, eth_p_ip, 9),
        // 8: if r4 != eth_p_ipv6 goto PASS (target index 39: off = 39 - (8 + 1) = 30)
        BpfInsn::jne_imm(R4, eth_p_ipv6, 30),
        // === IPv6 Branch (indices 9..16) ===
        // 9: r4 = r2
        BpfInsn::mov64_reg(R4, R2),
        // 10: r4 += 62 (14 eth + 40 ip6 + 8 udp)
        BpfInsn::add64_imm(R4, 62),
        // 11: if r4 > r3 goto PASS (target index 39: off = 39 - (11 + 1) = 27)
        BpfInsn::jgt_reg(R4, R3, 27),
        // 12: r4 = *(u8 *)(r2 + 20) (ip6->nexthdr at 14 + 6 = 20)
        BpfInsn::ldx_mem_b(R4, R2, 20),
        // 13: if r4 != 17 (IPPROTO_UDP) goto PASS (target index 39: off = 39 - (13 + 1) = 25)
        BpfInsn::jne_imm(R4, 17, 25),
        // 14: r4 = *(u16 *)(r2 + 56) (udp->dest at 14 + 40 + 2 = 56)
        BpfInsn::ldx_mem_h(R4, R2, 56),
        // 15: if r4 != udp_port goto PASS (target index 39: off = 39 - (15 + 1) = 23)
        BpfInsn::jne_imm(R4, udp_port, 23),
        // 16: goto REDIRECT (target index 33: off = 33 - (16 + 1) = 16)
        BpfInsn::ja(16),
        // === IPv4 Branch (indices 17..32) ===
        // 17: r4 = r2
        BpfInsn::mov64_reg(R4, R2),
        // 18: r4 += 42 (14 eth + 20 ip + 8 udp min bounds)
        BpfInsn::add64_imm(R4, 42),
        // 19: if r4 > r3 goto PASS (target index 39: off = 39 - (19 + 1) = 19)
        BpfInsn::jgt_reg(R4, R3, 19),
        // 20: r4 = *(u8 *)(r2 + 23) (ip->protocol at 14 + 9 = 23)
        BpfInsn::ldx_mem_b(R4, R2, 23),
        // 21: if r4 != 17 (IPPROTO_UDP) goto PASS (target index 39: off = 39 - (21 + 1) = 17)
        BpfInsn::jne_imm(R4, 17, 17),
        // 22: r5 = *(u8 *)(r2 + 14) (ip->v_ihl at offset 14)
        BpfInsn::ldx_mem_b(R5, R2, 14),
        // 23: r5 &= 0x0f (extract IHL)
        BpfInsn::and64_imm(R5, 0x0f),
        // 24: r5 <<= 2 (IHL in bytes: ihl * 4)
        BpfInsn::lsh64_imm(R5, 2),
        // 25: r4 = r2 (data)
        BpfInsn::mov64_reg(R4, R2),
        // 26: r4 += 14 (Ethernet header length)
        BpfInsn::add64_imm(R4, 14),
        // 27: r4 += r5 (r4 = data + 14 + ihl, pointing to start of UDP header)
        BpfInsn::add64_reg(R4, R5),
        // 28: r5 = r4
        BpfInsn::mov64_reg(R5, R4),
        // 29: r5 += 8 (UDP header length)
        BpfInsn::add64_imm(R5, 8),
        // 30: if r5 > r3 goto PASS (target index 39: off = 39 - (30 + 1) = 8)
        BpfInsn::jgt_reg(R5, R3, 8),
        // 31: r4 = *(u16 *)(r4 + 2) (udp->dest port)
        BpfInsn::ldx_mem_h(R4, R4, 2),
        // 32: if r4 != udp_port goto PASS (target index 39: off = 39 - (32 + 1) = 6)
        BpfInsn::jne_imm(R4, udp_port, 6),
        // === LABEL_REDIRECT (indices 33..38) ===
        // 33, 34: r1 = xsk_map_fd (BPF_LD_IMM64 pseudo map fd)
        BpfInsn::new(
            BPF_LD | BPF_DW | BPF_IMM,
            R1,
            BPF_PSEUDO_MAP_FD,
            0,
            xsk_map_fd,
        ),
        BpfInsn::new(0, 0, 0, 0, 0),
        // 35: r2 = *(u32 *)(r6 + 16) (ctx->rx_queue_index)
        BpfInsn::ldx_mem_w(R2, R6, 16),
        // 36: r3 = 0 (flags)
        BpfInsn::mov64_imm(R3, 0),
        // 37: call bpf_redirect_map(r1, r2, r3) -> r0
        BpfInsn::call_helper(BPF_FUNC_REDIRECT_MAP),
        // 38: exit (returns r0)
        BpfInsn::exit(),
        // === LABEL_PASS (indices 39..40) ===
        // 39: r0 = XDP_PASS (2)
        BpfInsn::mov64_imm(R0, XDP_PASS),
        // 40: exit
        BpfInsn::exit(),
    ]
}

/// Compiles and loads an eBPF XDP filter redirecting dual-stack UDP traffic for `listen_port` to `xsk_map_fd`.
pub fn bpf_prog_load_xdp(listen_port: u16, xsk_map_fd: RawFd) -> io::Result<RawFd> {
    let insns = bpf_prog_assemble_xdp(listen_port, xsk_map_fd);
    let license = b"GPL\0";

    let mut attr = BpfAttr { pad: [0u8; 152] };
    attr.prog_load = BpfProgLoadAttr {
        prog_type: BPF_PROG_TYPE_XDP,
        insn_cnt: insns.len() as u32,
        insns: insns.as_ptr() as u64,
        license: license.as_ptr() as u64,
        log_level: 0,
        log_size: 0,
        log_buf: 0,
        kern_version: 0,
        prog_flags: 0,
    };

    // SAFETY: sys_bpf is called with BPF_PROG_LOAD, attr points to valid stack-local
    // BpfProgLoadAttr, insns and license buffers outlive the syscall, and size matches sizeof(BpfAttr).
    let res = unsafe {
        sys_bpf(
            BPF_PROG_LOAD,
            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
            std::mem::size_of::<BpfAttr>(),
        )
    };

    if res < 0 {
        return Err(io::Error::last_os_error());
    }
    Ok(res as RawFd)
}

/// Fallback netlink-based attachment for older Linux kernels.
fn netlink_attach_xdp(ifindex: u32, prog_fd: RawFd) -> io::Result<RawFd> {
    // SAFETY: Invokes socket() to allocate an AF_NETLINK socket for ROUTE subsystem.
    let nl_sock = unsafe {
        libc::socket(
            libc::AF_NETLINK,
            libc::SOCK_RAW | libc::SOCK_CLOEXEC,
            libc::NETLINK_ROUTE,
        )
    };
    if nl_sock < 0 {
        return Err(io::Error::last_os_error());
    }
    let _sock_guard = AutoCloseFd(nl_sock);

    #[repr(C)]
    struct NlReq {
        nlh: libc::nlmsghdr,
        ifi: libc::ifinfomsg,
        xdp_hdr: libc::nlattr,
        fd_hdr: libc::nlattr,
        fd_val: i32,
    }

    let fd_attr_len = std::mem::size_of::<libc::nlattr>() + std::mem::size_of::<i32>();
    let xdp_attr_len = std::mem::size_of::<libc::nlattr>() + fd_attr_len;
    let total_len = std::mem::size_of::<libc::nlmsghdr>()
        + std::mem::size_of::<libc::ifinfomsg>()
        + xdp_attr_len;

    // SAFETY: Zeroed NlReq POD structure is safely formatted before sending.
    let mut req: NlReq = unsafe { std::mem::zeroed() };
    req.nlh.nlmsg_len = total_len as u32;
    req.nlh.nlmsg_type = libc::RTM_SETLINK;
    req.nlh.nlmsg_flags = (libc::NLM_F_REQUEST | libc::NLM_F_ACK) as u16;
    req.nlh.nlmsg_seq = 1;
    req.ifi.ifi_family = libc::AF_UNSPEC as u8;
    req.ifi.ifi_index = ifindex as i32;

    req.xdp_hdr.nla_len = xdp_attr_len as u16;
    req.xdp_hdr.nla_type = libc::NLA_F_NESTED as u16 | 43; // 43 = IFLA_XDP

    req.fd_hdr.nla_len = fd_attr_len as u16;
    req.fd_hdr.nla_type = 1; // 1 = IFLA_XDP_FD
    req.fd_val = prog_fd;

    // SAFETY: Sending exactly total_len bytes from stack-local req to valid open netlink socket.
    let send_ret = unsafe {
        libc::send(
            nl_sock,
            std::ptr::addr_of!(req).cast::<libc::c_void>(),
            total_len,
            0,
        )
    };
    if send_ret < 0 {
        return Err(io::Error::last_os_error());
    }

    let mut buf = [0u8; 256];
    // SAFETY: Receiving netlink reply into stack-allocated buffer.
    let recv_ret = unsafe {
        libc::recv(
            nl_sock,
            buf.as_mut_ptr().cast::<libc::c_void>(),
            buf.len(),
            0,
        )
    };
    if recv_ret < 0 {
        return Err(io::Error::last_os_error());
    }

    if recv_ret as usize
        >= std::mem::size_of::<libc::nlmsghdr>() + std::mem::size_of::<libc::nlmsgerr>()
    {
        // SAFETY: Byte offsets are bounds-checked by recv_ret check above; read_unaligned prevents alignment UB.
        let nlh = unsafe { std::ptr::read_unaligned(buf.as_ptr().cast::<libc::nlmsghdr>()) };
        if nlh.nlmsg_type == libc::NLMSG_ERROR as u16 {
            // SAFETY: Byte offset arithmetic strictly within buffer bounds; read_unaligned avoids alignment preconditions.
            let err = unsafe {
                std::ptr::read_unaligned(
                    buf.as_ptr()
                        .add(std::mem::size_of::<libc::nlmsghdr>())
                        .cast::<libc::nlmsgerr>(),
                )
            };
            if err.error != 0 {
                return Err(io::Error::from_raw_os_error(-err.error));
            }
        }
    }

    Ok(prog_fd)
}

/// Attaches an XDP program to the network interface specified by `ifindex`.
///
/// Tries modern `BPF_LINK_CREATE` first, falling back to netlink `RTM_SETLINK` if unsupported.
pub fn bpf_attach_xdp(ifindex: u32, prog_fd: RawFd) -> io::Result<RawFd> {
    // 1. Try BPF_LINK_CREATE (Linux 5.7+)
    let mut attr = BpfAttr { pad: [0u8; 152] };
    attr.link_create = BpfLinkCreateAttr {
        prog_fd: prog_fd as u32,
        target_ifindex: ifindex,
        attach_type: BPF_XDP,
        flags: 0,
    };

    // SAFETY: sys_bpf called with BPF_LINK_CREATE and valid initialized link_create attributes.
    let res = unsafe {
        sys_bpf(
            BPF_LINK_CREATE,
            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
            std::mem::size_of::<BpfAttr>(),
        )
    };

    if res >= 0 {
        return Ok(res as RawFd);
    }

    let link_err = io::Error::last_os_error();
    match link_err.raw_os_error() {
        Some(libc::EINVAL | libc::ENOSYS | libc::EOPNOTSUPP) => {
            // Fallback to Netlink RTM_SETLINK
            netlink_attach_xdp(ifindex, prog_fd)
        }
        _ => Err(link_err),
    }
}

/// RAII helper ensuring raw file descriptors are closed if initialization fails early.
pub(crate) struct AutoCloseFd(pub RawFd);

impl AutoCloseFd {
    pub fn into_raw(mut self) -> RawFd {
        let fd = self.0;
        self.0 = -1;
        fd
    }
}

impl Drop for AutoCloseFd {
    fn drop(&mut self) {
        if self.0 >= 0 {
            // SAFETY: Closing open file descriptor exclusively owned by AutoCloseFd.
            unsafe {
                libc::close(self.0);
            }
        }
    }
}

/// RAII container for loaded eBPF redirect map and XDP program.
pub struct XdpRedirectFilter {
    map_fd: RawFd,
    prog_fd: RawFd,
}

impl XdpRedirectFilter {
    /// Constructs a new `XdpRedirectFilter` from existing raw map and program file descriptors.
    pub fn new(map_fd: RawFd, prog_fd: RawFd) -> Self {
        Self { map_fd, prog_fd }
    }

    /// File descriptor of the underlying `BPF_MAP_TYPE_XSKMAP`.
    pub fn map_fd(&self) -> RawFd {
        self.map_fd
    }

    /// File descriptor of the loaded eBPF XDP redirect program.
    pub fn prog_fd(&self) -> RawFd {
        self.prog_fd
    }

    /// Consumes the filter container, returning raw file descriptors `(map_fd, prog_fd)`
    /// without closing them.
    pub fn into_raw(mut self) -> (RawFd, RawFd) {
        let map_fd = self.map_fd;
        let prog_fd = self.prog_fd;
        self.map_fd = -1;
        self.prog_fd = -1;
        (map_fd, prog_fd)
    }

    /// Dynamically sets or updates the AF_XDP socket file descriptor for a specific queue ID in the map.
    pub fn set_socket_for_queue(&mut self, queue_id: u32, xsk_fd: RawFd) -> io::Result<()> {
        if self.map_fd < 0 {
            return Err(io::Error::from_raw_os_error(libc::EBADF));
        }
        if xsk_fd < 0 {
            return Err(io::Error::from_raw_os_error(libc::EBADF));
        }
        bpf_map_update_xsk(self.map_fd, queue_id, xsk_fd)
    }

    /// Loads and attaches a multi-queue XDP redirect filter to `ifname` for `listen_port`
    /// across the specified `queue_fds`.
    pub fn load_multi_queue(
        ifname: &str,
        listen_port: u16,
        queue_fds: &[(u32, RawFd)],
    ) -> Result<Self, BpfFilterStatus> {
        let c_ifname = match CString::new(ifname) {
            Ok(c) => c,
            Err(_) => return Err(BpfFilterStatus::Unsupported),
        };

        // SAFETY: c_ifname is a valid null-terminated C string.
        let ifindex = unsafe { libc::if_nametoindex(c_ifname.as_ptr()) };
        if ifindex == 0 {
            return Err(BpfFilterStatus::Unsupported);
        }

        // 1. Create BPF_MAP_TYPE_XSKMAP dimensioned to hold all specified queues.
        let max_queue_id = queue_fds.iter().map(|(q, _)| *q).max().unwrap_or(0);
        let max_entries = max_queue_id
            .checked_add(1)
            .unwrap_or(64)
            .max(queue_fds.len() as u32)
            .max(64);

        let map_fd = match bpf_map_create_xsk(max_entries) {
            Ok(fd) => fd,
            Err(e) => {
                return match e.raw_os_error() {
                    Some(libc::EPERM | libc::EACCES) => Err(BpfFilterStatus::FallbackUnprivileged),
                    _ => Err(BpfFilterStatus::Unsupported),
                };
            }
        };
        let map_guard = AutoCloseFd(map_fd);

        // 2. Populate each (queue_id, xsk_fd) entry into the map if valid.
        for &(queue_id, xsk_fd) in queue_fds {
            if xsk_fd >= 0 {
                if let Err(e) = bpf_map_update_xsk(map_fd, queue_id, xsk_fd) {
                    return match e.raw_os_error() {
                        Some(libc::EPERM | libc::EACCES) => {
                            Err(BpfFilterStatus::FallbackUnprivileged)
                        }
                        _ => Err(BpfFilterStatus::Unsupported),
                    };
                }
            } else {
                // Negative xsk_fd is invalid for map update
                return Err(BpfFilterStatus::Unsupported);
            }
        }

        // 3. Load XDP program
        let prog_fd = match bpf_prog_load_xdp(listen_port, map_fd) {
            Ok(fd) => fd,
            Err(e) => {
                return match e.raw_os_error() {
                    Some(libc::EPERM | libc::EACCES) => Err(BpfFilterStatus::FallbackUnprivileged),
                    _ => Err(BpfFilterStatus::Unsupported),
                };
            }
        };
        let prog_guard = AutoCloseFd(prog_fd);

        // 4. Attach XDP program to interface
        if let Err(e) = bpf_attach_xdp(ifindex, prog_fd) {
            return match e.raw_os_error() {
                Some(libc::EPERM | libc::EACCES) => Err(BpfFilterStatus::FallbackUnprivileged),
                _ => Err(BpfFilterStatus::Unsupported),
            };
        }

        let map_fd = map_guard.into_raw();
        let prog_fd = prog_guard.into_raw();
        Ok(Self { map_fd, prog_fd })
    }

    /// Loads and attaches an XDP redirect filter to `ifname` for `listen_port` on a single queue.
    pub fn load_and_attach(
        ifname: &str,
        listen_port: u16,
        queue_id: u32,
        xsk_fd: RawFd,
    ) -> Result<Self, BpfFilterStatus> {
        Self::load_multi_queue(ifname, listen_port, &[(queue_id, xsk_fd)])
    }

    /// Opportunistically attempts to attach the eBPF XDP redirect filter across multiple queues.
    ///
    /// Degrades fail-soft to `FallbackUnprivileged` or `Unsupported` without panicking.
    pub fn attach_multi_queue(
        ifname: &str,
        listen_port: u16,
        queue_fds: &[(u32, RawFd)],
    ) -> BpfFilterStatus {
        match Self::load_multi_queue(ifname, listen_port, queue_fds) {
            Ok(filter) => {
                let (map_fd, prog_fd) = filter.into_raw();
                if map_fd >= 0 {
                    // SAFETY: Closing unexported map descriptor; kernel keeps map reference alive via BPF prog.
                    unsafe {
                        libc::close(map_fd);
                    }
                }
                BpfFilterStatus::Attached(prog_fd)
            }
            Err(status) => status,
        }
    }

    /// Opportunistically attempts to attach the eBPF XDP redirect filter on a single queue.
    ///
    /// Degrades fail-soft to `FallbackUnprivileged` or `Unsupported` without panicking.
    pub fn attach_opportunistic(
        ifname: &str,
        listen_port: u16,
        queue_id: u32,
        xsk_fd: RawFd,
    ) -> BpfFilterStatus {
        Self::attach_multi_queue(ifname, listen_port, &[(queue_id, xsk_fd)])
    }
}

impl Drop for XdpRedirectFilter {
    fn drop(&mut self) {
        if self.map_fd >= 0 {
            // SAFETY: self.map_fd is an open file descriptor owned by XdpRedirectFilter.
            // Closing it releases the kernel BPF map resource.
            unsafe {
                libc::close(self.map_fd);
            }
            self.map_fd = -1;
        }
        if self.prog_fd >= 0 {
            // SAFETY: self.prog_fd is an open file descriptor owned by XdpRedirectFilter.
            // Closing it releases userspace reference to the BPF program.
            unsafe {
                libc::close(self.prog_fd);
            }
            self.prog_fd = -1;
        }
    }
}