yip_io/
bpf.rs

1//! Self-contained eBPF XSK redirect driver and map loader.
2//!
3//! Provides a zero-dependency Linux eBPF/XDP driver using direct `bpf(2)` syscalls
4//! to direct incoming tunnel UDP datagrams straight into AF_XDP rings (`BPF_MAP_TYPE_XSKMAP`)
5//! at the NIC driver layer, bypassing `sk_buff` allocations.
6//!
7//! All raw syscalls and pointer manipulations are quarantined within this module,
8//! with explicit `// SAFETY:` justifications on every unsafe block.
9
10use std::ffi::CString;
11use std::io;
12use std::os::fd::RawFd;
13
14/// Operational status of the opportunistic eBPF XDP filter attachment.
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub enum BpfFilterStatus {
17    /// Filter successfully loaded and attached to the network interface.
18    Attached(RawFd),
19    /// Unprivileged execution (missing `CAP_BPF` / `CAP_NET_ADMIN` / `EPERM`).
20    FallbackUnprivileged,
21    /// BPF or XDP unsupported on the interface or kernel.
22    Unsupported,
23}
24
25// BPF syscall command constants
26pub const BPF_MAP_CREATE: libc::c_int = 0;
27pub const BPF_MAP_UPDATE_ELEM: libc::c_int = 2;
28pub const BPF_PROG_LOAD: libc::c_int = 5;
29pub const BPF_LINK_CREATE: libc::c_int = 28;
30
31// BPF types
32pub const BPF_MAP_TYPE_XSKMAP: u32 = 17;
33pub const BPF_PROG_TYPE_XDP: u32 = 6;
34pub const BPF_XDP: u32 = 37;
35
36// BPF helper function IDs
37pub const BPF_FUNC_REDIRECT_MAP: i32 = 51;
38
39// XDP return actions
40pub const XDP_PASS: i32 = 2;
41
42// Pseudo map FD flag for BPF_LD_IMM64
43pub const BPF_PSEUDO_MAP_FD: u8 = 1;
44
45// eBPF instruction encoding constants
46pub const BPF_LD: u8 = 0x00;
47pub const BPF_LDX: u8 = 0x01;
48pub const BPF_ALU64: u8 = 0x07;
49pub const BPF_JMP: u8 = 0x05;
50
51pub const BPF_W: u8 = 0x00;
52pub const BPF_H: u8 = 0x08;
53pub const BPF_B: u8 = 0x10;
54pub const BPF_DW: u8 = 0x18;
55
56pub const BPF_IMM: u8 = 0x00;
57pub const BPF_MEM: u8 = 0x60;
58
59pub const BPF_K: u8 = 0x00;
60pub const BPF_X: u8 = 0x08;
61
62pub const BPF_ADD: u8 = 0x00;
63pub const BPF_AND: u8 = 0x50;
64pub const BPF_LSH: u8 = 0x60;
65pub const BPF_MOV: u8 = 0xb0;
66
67pub const BPF_JA: u8 = 0x00;
68pub const BPF_JEQ: u8 = 0x10;
69pub const BPF_JGT: u8 = 0x20;
70pub const BPF_JNE: u8 = 0x50;
71pub const BPF_CALL: u8 = 0x80;
72pub const BPF_EXIT: u8 = 0x90;
73
74// Registers
75pub const R0: u8 = 0;
76pub const R1: u8 = 1;
77pub const R2: u8 = 2;
78pub const R3: u8 = 3;
79pub const R4: u8 = 4;
80pub const R5: u8 = 5;
81pub const R6: u8 = 6;
82
83/// 8-byte eBPF instruction binary layout matching Linux kernel ABI (`struct bpf_insn`).
84#[repr(C)]
85#[derive(Clone, Copy, Debug, PartialEq, Eq)]
86pub struct BpfInsn {
87    pub code: u8,
88    pub regs: u8, // (src_reg << 4) | (dst_reg & 0x0f)
89    pub off: i16,
90    pub imm: i32,
91}
92
93impl BpfInsn {
94    pub const fn new(code: u8, dst: u8, src: u8, off: i16, imm: i32) -> Self {
95        Self {
96            code,
97            regs: ((src & 0x0f) << 4) | (dst & 0x0f),
98            off,
99            imm,
100        }
101    }
102
103    pub const fn mov64_reg(dst: u8, src: u8) -> Self {
104        Self::new(BPF_ALU64 | BPF_MOV | BPF_X, dst, src, 0, 0)
105    }
106
107    pub const fn mov64_imm(dst: u8, imm: i32) -> Self {
108        Self::new(BPF_ALU64 | BPF_MOV | BPF_K, dst, 0, 0, imm)
109    }
110
111    pub const fn add64_imm(dst: u8, imm: i32) -> Self {
112        Self::new(BPF_ALU64 | BPF_ADD | BPF_K, dst, 0, 0, imm)
113    }
114
115    pub const fn add64_reg(dst: u8, src: u8) -> Self {
116        Self::new(BPF_ALU64 | BPF_ADD | BPF_X, dst, src, 0, 0)
117    }
118
119    pub const fn and64_imm(dst: u8, imm: i32) -> Self {
120        Self::new(BPF_ALU64 | BPF_AND | BPF_K, dst, 0, 0, imm)
121    }
122
123    pub const fn lsh64_imm(dst: u8, imm: i32) -> Self {
124        Self::new(BPF_ALU64 | BPF_LSH | BPF_K, dst, 0, 0, imm)
125    }
126
127    pub const fn ja(off: i16) -> Self {
128        Self::new(BPF_JMP | BPF_JA, 0, 0, off, 0)
129    }
130
131    pub const fn jeq_imm(dst: u8, imm: i32, off: i16) -> Self {
132        Self::new(BPF_JMP | BPF_JEQ | BPF_K, dst, 0, off, imm)
133    }
134
135    pub const fn ldx_mem_w(dst: u8, src: u8, off: i16) -> Self {
136        Self::new(BPF_LDX | BPF_MEM | BPF_W, dst, src, off, 0)
137    }
138
139    pub const fn ldx_mem_h(dst: u8, src: u8, off: i16) -> Self {
140        Self::new(BPF_LDX | BPF_MEM | BPF_H, dst, src, off, 0)
141    }
142
143    pub const fn ldx_mem_b(dst: u8, src: u8, off: i16) -> Self {
144        Self::new(BPF_LDX | BPF_MEM | BPF_B, dst, src, off, 0)
145    }
146
147    pub const fn jgt_reg(dst: u8, src: u8, off: i16) -> Self {
148        Self::new(BPF_JMP | BPF_JGT | BPF_X, dst, src, off, 0)
149    }
150
151    pub const fn jne_imm(dst: u8, imm: i32, off: i16) -> Self {
152        Self::new(BPF_JMP | BPF_JNE | BPF_K, dst, 0, off, imm)
153    }
154
155    pub const fn call_helper(func: i32) -> Self {
156        Self::new(BPF_JMP | BPF_CALL, 0, 0, 0, func)
157    }
158
159    pub const fn exit() -> Self {
160        Self::new(BPF_JMP | BPF_EXIT, 0, 0, 0, 0)
161    }
162}
163
164#[repr(C)]
165#[derive(Clone, Copy)]
166pub struct BpfMapCreateAttr {
167    pub map_type: u32,
168    pub key_size: u32,
169    pub value_size: u32,
170    pub max_entries: u32,
171    pub map_flags: u32,
172}
173
174#[repr(C)]
175#[derive(Clone, Copy)]
176pub struct BpfMapElemAttr {
177    pub map_fd: u32,
178    pub _pad: u32,
179    pub key: u64,
180    pub value: u64,
181    pub flags: u64,
182}
183
184#[repr(C)]
185#[derive(Clone, Copy)]
186pub struct BpfProgLoadAttr {
187    pub prog_type: u32,
188    pub insn_cnt: u32,
189    pub insns: u64,
190    pub license: u64,
191    pub log_level: u32,
192    pub log_size: u32,
193    pub log_buf: u64,
194    pub kern_version: u32,
195    pub prog_flags: u32,
196}
197
198#[repr(C)]
199#[derive(Clone, Copy)]
200pub struct BpfLinkCreateAttr {
201    pub prog_fd: u32,
202    pub target_ifindex: u32,
203    pub attach_type: u32,
204    pub flags: u32,
205}
206
207/// Linux `union bpf_attr` representation, 152 bytes.
208#[repr(C)]
209pub union BpfAttr {
210    pub map_create: BpfMapCreateAttr,
211    pub map_elem: BpfMapElemAttr,
212    pub prog_load: BpfProgLoadAttr,
213    pub link_create: BpfLinkCreateAttr,
214    pub pad: [u8; 152],
215}
216
217/// Raw syscall wrapper for Linux `bpf(2)`.
218///
219/// # Safety
220///
221/// Caller must ensure `attr` points to an initialized memory buffer of at least `size`
222/// bytes matching the kernel's ABI for command `cmd`, or NULL if the command expects no attributes.
223pub unsafe fn sys_bpf(cmd: libc::c_int, attr: *const libc::c_void, size: usize) -> libc::c_long {
224    // SAFETY: Invokes the Linux bpf syscall with caller-verified arguments.
225    libc::syscall(libc::SYS_bpf, cmd, attr, size)
226}
227
228/// Creates a `BPF_MAP_TYPE_XSKMAP` map holding up to `max_entries` AF_XDP socket descriptors.
229pub fn bpf_map_create_xsk(max_entries: u32) -> io::Result<RawFd> {
230    let mut attr = BpfAttr { pad: [0u8; 152] };
231    attr.map_create = BpfMapCreateAttr {
232        map_type: BPF_MAP_TYPE_XSKMAP,
233        key_size: 4,
234        value_size: 4,
235        max_entries,
236        map_flags: 0,
237    };
238
239    // SAFETY: sys_bpf is called with BPF_MAP_CREATE, a valid pointer to stack-local
240    // BpfAttr, and exact size of BpfAttr matching the kernel ABI.
241    let res = unsafe {
242        sys_bpf(
243            BPF_MAP_CREATE,
244            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
245            std::mem::size_of::<BpfAttr>(),
246        )
247    };
248
249    if res < 0 {
250        return Err(io::Error::last_os_error());
251    }
252    Ok(res as RawFd)
253}
254
255/// Updates an entry in an XSK map mapping `queue_id` to `xsk_fd`.
256pub fn bpf_map_update_xsk(map_fd: RawFd, queue_id: u32, xsk_fd: RawFd) -> io::Result<()> {
257    let mut attr = BpfAttr { pad: [0u8; 152] };
258    let key = queue_id;
259    let value = xsk_fd as u32;
260
261    attr.map_elem = BpfMapElemAttr {
262        map_fd: map_fd as u32,
263        _pad: 0,
264        key: std::ptr::addr_of!(key) as u64,
265        value: std::ptr::addr_of!(value) as u64,
266        flags: 0,
267    };
268
269    // SAFETY: sys_bpf is called with BPF_MAP_UPDATE_ELEM, attr points to valid
270    // initialized BpfMapElemAttr, key and value point to valid stack locals that
271    // outlive the syscall, and size matches sizeof(BpfAttr).
272    let res = unsafe {
273        sys_bpf(
274            BPF_MAP_UPDATE_ELEM,
275            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
276            std::mem::size_of::<BpfAttr>(),
277        )
278    };
279
280    if res < 0 {
281        return Err(io::Error::last_os_error());
282    }
283    Ok(())
284}
285
286/// Assembles the dual-stack outer UDP eBPF XDP filter bytecode.
287///
288/// Matches both IPv4 (`0x0800`) and IPv6 (`0x86dd`) outer UDP datagrams targeting `listen_port`,
289/// steering matched packets directly to core-pinned AF_XDP rings via `bpf_redirect_map(xsk_map_fd, ctx->rx_queue_index, 0)`.
290pub fn bpf_prog_assemble_xdp(listen_port: u16, xsk_map_fd: RawFd) -> Vec<BpfInsn> {
291    let eth_p_ip = (0x0800u16).to_be() as i32;
292    let eth_p_ipv6 = (0x86ddu16).to_be() as i32;
293    let udp_port = listen_port.to_be() as i32;
294
295    vec![
296        // 0: r6 = r1 (save context pointer)
297        BpfInsn::mov64_reg(R6, R1),
298        // 1: r2 = *(u32 *)(r6 + 0) (ctx->data)
299        BpfInsn::ldx_mem_w(R2, R6, 0),
300        // 2: r3 = *(u32 *)(r6 + 4) (ctx->data_end)
301        BpfInsn::ldx_mem_w(R3, R6, 4),
302        // 3: r4 = r2
303        BpfInsn::mov64_reg(R4, R2),
304        // 4: r4 += 14 (Ethernet header length)
305        BpfInsn::add64_imm(R4, 14),
306        // 5: if r4 > r3 goto PASS (target index 39: off = 39 - (5 + 1) = 33)
307        BpfInsn::jgt_reg(R4, R3, 33),
308        // 6: r4 = *(u16 *)(r2 + 12) (eth->h_proto)
309        BpfInsn::ldx_mem_h(R4, R2, 12),
310        // 7: if r4 == eth_p_ip goto IPV4 (target index 17: off = 17 - (7 + 1) = 9)
311        BpfInsn::jeq_imm(R4, eth_p_ip, 9),
312        // 8: if r4 != eth_p_ipv6 goto PASS (target index 39: off = 39 - (8 + 1) = 30)
313        BpfInsn::jne_imm(R4, eth_p_ipv6, 30),
314        // === IPv6 Branch (indices 9..16) ===
315        // 9: r4 = r2
316        BpfInsn::mov64_reg(R4, R2),
317        // 10: r4 += 62 (14 eth + 40 ip6 + 8 udp)
318        BpfInsn::add64_imm(R4, 62),
319        // 11: if r4 > r3 goto PASS (target index 39: off = 39 - (11 + 1) = 27)
320        BpfInsn::jgt_reg(R4, R3, 27),
321        // 12: r4 = *(u8 *)(r2 + 20) (ip6->nexthdr at 14 + 6 = 20)
322        BpfInsn::ldx_mem_b(R4, R2, 20),
323        // 13: if r4 != 17 (IPPROTO_UDP) goto PASS (target index 39: off = 39 - (13 + 1) = 25)
324        BpfInsn::jne_imm(R4, 17, 25),
325        // 14: r4 = *(u16 *)(r2 + 56) (udp->dest at 14 + 40 + 2 = 56)
326        BpfInsn::ldx_mem_h(R4, R2, 56),
327        // 15: if r4 != udp_port goto PASS (target index 39: off = 39 - (15 + 1) = 23)
328        BpfInsn::jne_imm(R4, udp_port, 23),
329        // 16: goto REDIRECT (target index 33: off = 33 - (16 + 1) = 16)
330        BpfInsn::ja(16),
331        // === IPv4 Branch (indices 17..32) ===
332        // 17: r4 = r2
333        BpfInsn::mov64_reg(R4, R2),
334        // 18: r4 += 42 (14 eth + 20 ip + 8 udp min bounds)
335        BpfInsn::add64_imm(R4, 42),
336        // 19: if r4 > r3 goto PASS (target index 39: off = 39 - (19 + 1) = 19)
337        BpfInsn::jgt_reg(R4, R3, 19),
338        // 20: r4 = *(u8 *)(r2 + 23) (ip->protocol at 14 + 9 = 23)
339        BpfInsn::ldx_mem_b(R4, R2, 23),
340        // 21: if r4 != 17 (IPPROTO_UDP) goto PASS (target index 39: off = 39 - (21 + 1) = 17)
341        BpfInsn::jne_imm(R4, 17, 17),
342        // 22: r5 = *(u8 *)(r2 + 14) (ip->v_ihl at offset 14)
343        BpfInsn::ldx_mem_b(R5, R2, 14),
344        // 23: r5 &= 0x0f (extract IHL)
345        BpfInsn::and64_imm(R5, 0x0f),
346        // 24: r5 <<= 2 (IHL in bytes: ihl * 4)
347        BpfInsn::lsh64_imm(R5, 2),
348        // 25: r4 = r2 (data)
349        BpfInsn::mov64_reg(R4, R2),
350        // 26: r4 += 14 (Ethernet header length)
351        BpfInsn::add64_imm(R4, 14),
352        // 27: r4 += r5 (r4 = data + 14 + ihl, pointing to start of UDP header)
353        BpfInsn::add64_reg(R4, R5),
354        // 28: r5 = r4
355        BpfInsn::mov64_reg(R5, R4),
356        // 29: r5 += 8 (UDP header length)
357        BpfInsn::add64_imm(R5, 8),
358        // 30: if r5 > r3 goto PASS (target index 39: off = 39 - (30 + 1) = 8)
359        BpfInsn::jgt_reg(R5, R3, 8),
360        // 31: r4 = *(u16 *)(r4 + 2) (udp->dest port)
361        BpfInsn::ldx_mem_h(R4, R4, 2),
362        // 32: if r4 != udp_port goto PASS (target index 39: off = 39 - (32 + 1) = 6)
363        BpfInsn::jne_imm(R4, udp_port, 6),
364        // === LABEL_REDIRECT (indices 33..38) ===
365        // 33, 34: r1 = xsk_map_fd (BPF_LD_IMM64 pseudo map fd)
366        BpfInsn::new(
367            BPF_LD | BPF_DW | BPF_IMM,
368            R1,
369            BPF_PSEUDO_MAP_FD,
370            0,
371            xsk_map_fd,
372        ),
373        BpfInsn::new(0, 0, 0, 0, 0),
374        // 35: r2 = *(u32 *)(r6 + 16) (ctx->rx_queue_index)
375        BpfInsn::ldx_mem_w(R2, R6, 16),
376        // 36: r3 = 0 (flags)
377        BpfInsn::mov64_imm(R3, 0),
378        // 37: call bpf_redirect_map(r1, r2, r3) -> r0
379        BpfInsn::call_helper(BPF_FUNC_REDIRECT_MAP),
380        // 38: exit (returns r0)
381        BpfInsn::exit(),
382        // === LABEL_PASS (indices 39..40) ===
383        // 39: r0 = XDP_PASS (2)
384        BpfInsn::mov64_imm(R0, XDP_PASS),
385        // 40: exit
386        BpfInsn::exit(),
387    ]
388}
389
390/// Compiles and loads an eBPF XDP filter redirecting dual-stack UDP traffic for `listen_port` to `xsk_map_fd`.
391pub fn bpf_prog_load_xdp(listen_port: u16, xsk_map_fd: RawFd) -> io::Result<RawFd> {
392    let insns = bpf_prog_assemble_xdp(listen_port, xsk_map_fd);
393    let license = b"GPL\0";
394
395    let mut attr = BpfAttr { pad: [0u8; 152] };
396    attr.prog_load = BpfProgLoadAttr {
397        prog_type: BPF_PROG_TYPE_XDP,
398        insn_cnt: insns.len() as u32,
399        insns: insns.as_ptr() as u64,
400        license: license.as_ptr() as u64,
401        log_level: 0,
402        log_size: 0,
403        log_buf: 0,
404        kern_version: 0,
405        prog_flags: 0,
406    };
407
408    // SAFETY: sys_bpf is called with BPF_PROG_LOAD, attr points to valid stack-local
409    // BpfProgLoadAttr, insns and license buffers outlive the syscall, and size matches sizeof(BpfAttr).
410    let res = unsafe {
411        sys_bpf(
412            BPF_PROG_LOAD,
413            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
414            std::mem::size_of::<BpfAttr>(),
415        )
416    };
417
418    if res < 0 {
419        return Err(io::Error::last_os_error());
420    }
421    Ok(res as RawFd)
422}
423
424/// Fallback netlink-based attachment for older Linux kernels.
425fn netlink_attach_xdp(ifindex: u32, prog_fd: RawFd) -> io::Result<RawFd> {
426    // SAFETY: Invokes socket() to allocate an AF_NETLINK socket for ROUTE subsystem.
427    let nl_sock = unsafe {
428        libc::socket(
429            libc::AF_NETLINK,
430            libc::SOCK_RAW | libc::SOCK_CLOEXEC,
431            libc::NETLINK_ROUTE,
432        )
433    };
434    if nl_sock < 0 {
435        return Err(io::Error::last_os_error());
436    }
437    let _sock_guard = AutoCloseFd(nl_sock);
438
439    #[repr(C)]
440    struct NlReq {
441        nlh: libc::nlmsghdr,
442        ifi: libc::ifinfomsg,
443        xdp_hdr: libc::nlattr,
444        fd_hdr: libc::nlattr,
445        fd_val: i32,
446    }
447
448    let fd_attr_len = std::mem::size_of::<libc::nlattr>() + std::mem::size_of::<i32>();
449    let xdp_attr_len = std::mem::size_of::<libc::nlattr>() + fd_attr_len;
450    let total_len = std::mem::size_of::<libc::nlmsghdr>()
451        + std::mem::size_of::<libc::ifinfomsg>()
452        + xdp_attr_len;
453
454    // SAFETY: Zeroed NlReq POD structure is safely formatted before sending.
455    let mut req: NlReq = unsafe { std::mem::zeroed() };
456    req.nlh.nlmsg_len = total_len as u32;
457    req.nlh.nlmsg_type = libc::RTM_SETLINK;
458    req.nlh.nlmsg_flags = (libc::NLM_F_REQUEST | libc::NLM_F_ACK) as u16;
459    req.nlh.nlmsg_seq = 1;
460    req.ifi.ifi_family = libc::AF_UNSPEC as u8;
461    req.ifi.ifi_index = ifindex as i32;
462
463    req.xdp_hdr.nla_len = xdp_attr_len as u16;
464    req.xdp_hdr.nla_type = libc::NLA_F_NESTED as u16 | 43; // 43 = IFLA_XDP
465
466    req.fd_hdr.nla_len = fd_attr_len as u16;
467    req.fd_hdr.nla_type = 1; // 1 = IFLA_XDP_FD
468    req.fd_val = prog_fd;
469
470    // SAFETY: Sending exactly total_len bytes from stack-local req to valid open netlink socket.
471    let send_ret = unsafe {
472        libc::send(
473            nl_sock,
474            std::ptr::addr_of!(req).cast::<libc::c_void>(),
475            total_len,
476            0,
477        )
478    };
479    if send_ret < 0 {
480        return Err(io::Error::last_os_error());
481    }
482
483    let mut buf = [0u8; 256];
484    // SAFETY: Receiving netlink reply into stack-allocated buffer.
485    let recv_ret = unsafe {
486        libc::recv(
487            nl_sock,
488            buf.as_mut_ptr().cast::<libc::c_void>(),
489            buf.len(),
490            0,
491        )
492    };
493    if recv_ret < 0 {
494        return Err(io::Error::last_os_error());
495    }
496
497    if recv_ret as usize
498        >= std::mem::size_of::<libc::nlmsghdr>() + std::mem::size_of::<libc::nlmsgerr>()
499    {
500        // SAFETY: Byte offsets are bounds-checked by recv_ret check above; read_unaligned prevents alignment UB.
501        let nlh = unsafe { std::ptr::read_unaligned(buf.as_ptr().cast::<libc::nlmsghdr>()) };
502        if nlh.nlmsg_type == libc::NLMSG_ERROR as u16 {
503            // SAFETY: Byte offset arithmetic strictly within buffer bounds; read_unaligned avoids alignment preconditions.
504            let err = unsafe {
505                std::ptr::read_unaligned(
506                    buf.as_ptr()
507                        .add(std::mem::size_of::<libc::nlmsghdr>())
508                        .cast::<libc::nlmsgerr>(),
509                )
510            };
511            if err.error != 0 {
512                return Err(io::Error::from_raw_os_error(-err.error));
513            }
514        }
515    }
516
517    Ok(prog_fd)
518}
519
520/// Attaches an XDP program to the network interface specified by `ifindex`.
521///
522/// Tries modern `BPF_LINK_CREATE` first, falling back to netlink `RTM_SETLINK` if unsupported.
523pub fn bpf_attach_xdp(ifindex: u32, prog_fd: RawFd) -> io::Result<RawFd> {
524    // 1. Try BPF_LINK_CREATE (Linux 5.7+)
525    let mut attr = BpfAttr { pad: [0u8; 152] };
526    attr.link_create = BpfLinkCreateAttr {
527        prog_fd: prog_fd as u32,
528        target_ifindex: ifindex,
529        attach_type: BPF_XDP,
530        flags: 0,
531    };
532
533    // SAFETY: sys_bpf called with BPF_LINK_CREATE and valid initialized link_create attributes.
534    let res = unsafe {
535        sys_bpf(
536            BPF_LINK_CREATE,
537            std::ptr::addr_of!(attr).cast::<libc::c_void>(),
538            std::mem::size_of::<BpfAttr>(),
539        )
540    };
541
542    if res >= 0 {
543        return Ok(res as RawFd);
544    }
545
546    let link_err = io::Error::last_os_error();
547    match link_err.raw_os_error() {
548        Some(libc::EINVAL | libc::ENOSYS | libc::EOPNOTSUPP) => {
549            // Fallback to Netlink RTM_SETLINK
550            netlink_attach_xdp(ifindex, prog_fd)
551        }
552        _ => Err(link_err),
553    }
554}
555
556/// RAII helper ensuring raw file descriptors are closed if initialization fails early.
557pub(crate) struct AutoCloseFd(pub RawFd);
558
559impl AutoCloseFd {
560    pub fn into_raw(mut self) -> RawFd {
561        let fd = self.0;
562        self.0 = -1;
563        fd
564    }
565}
566
567impl Drop for AutoCloseFd {
568    fn drop(&mut self) {
569        if self.0 >= 0 {
570            // SAFETY: Closing open file descriptor exclusively owned by AutoCloseFd.
571            unsafe {
572                libc::close(self.0);
573            }
574        }
575    }
576}
577
578/// RAII container for loaded eBPF redirect map and XDP program.
579pub struct XdpRedirectFilter {
580    map_fd: RawFd,
581    prog_fd: RawFd,
582}
583
584impl XdpRedirectFilter {
585    /// Constructs a new `XdpRedirectFilter` from existing raw map and program file descriptors.
586    pub fn new(map_fd: RawFd, prog_fd: RawFd) -> Self {
587        Self { map_fd, prog_fd }
588    }
589
590    /// File descriptor of the underlying `BPF_MAP_TYPE_XSKMAP`.
591    pub fn map_fd(&self) -> RawFd {
592        self.map_fd
593    }
594
595    /// File descriptor of the loaded eBPF XDP redirect program.
596    pub fn prog_fd(&self) -> RawFd {
597        self.prog_fd
598    }
599
600    /// Consumes the filter container, returning raw file descriptors `(map_fd, prog_fd)`
601    /// without closing them.
602    pub fn into_raw(mut self) -> (RawFd, RawFd) {
603        let map_fd = self.map_fd;
604        let prog_fd = self.prog_fd;
605        self.map_fd = -1;
606        self.prog_fd = -1;
607        (map_fd, prog_fd)
608    }
609
610    /// Dynamically sets or updates the AF_XDP socket file descriptor for a specific queue ID in the map.
611    pub fn set_socket_for_queue(&mut self, queue_id: u32, xsk_fd: RawFd) -> io::Result<()> {
612        if self.map_fd < 0 {
613            return Err(io::Error::from_raw_os_error(libc::EBADF));
614        }
615        if xsk_fd < 0 {
616            return Err(io::Error::from_raw_os_error(libc::EBADF));
617        }
618        bpf_map_update_xsk(self.map_fd, queue_id, xsk_fd)
619    }
620
621    /// Loads and attaches a multi-queue XDP redirect filter to `ifname` for `listen_port`
622    /// across the specified `queue_fds`.
623    pub fn load_multi_queue(
624        ifname: &str,
625        listen_port: u16,
626        queue_fds: &[(u32, RawFd)],
627    ) -> Result<Self, BpfFilterStatus> {
628        let c_ifname = match CString::new(ifname) {
629            Ok(c) => c,
630            Err(_) => return Err(BpfFilterStatus::Unsupported),
631        };
632
633        // SAFETY: c_ifname is a valid null-terminated C string.
634        let ifindex = unsafe { libc::if_nametoindex(c_ifname.as_ptr()) };
635        if ifindex == 0 {
636            return Err(BpfFilterStatus::Unsupported);
637        }
638
639        // 1. Create BPF_MAP_TYPE_XSKMAP dimensioned to hold all specified queues.
640        let max_queue_id = queue_fds.iter().map(|(q, _)| *q).max().unwrap_or(0);
641        let max_entries = max_queue_id
642            .checked_add(1)
643            .unwrap_or(64)
644            .max(queue_fds.len() as u32)
645            .max(64);
646
647        let map_fd = match bpf_map_create_xsk(max_entries) {
648            Ok(fd) => fd,
649            Err(e) => {
650                return match e.raw_os_error() {
651                    Some(libc::EPERM | libc::EACCES) => Err(BpfFilterStatus::FallbackUnprivileged),
652                    _ => Err(BpfFilterStatus::Unsupported),
653                };
654            }
655        };
656        let map_guard = AutoCloseFd(map_fd);
657
658        // 2. Populate each (queue_id, xsk_fd) entry into the map if valid.
659        for &(queue_id, xsk_fd) in queue_fds {
660            if xsk_fd >= 0 {
661                if let Err(e) = bpf_map_update_xsk(map_fd, queue_id, xsk_fd) {
662                    return match e.raw_os_error() {
663                        Some(libc::EPERM | libc::EACCES) => {
664                            Err(BpfFilterStatus::FallbackUnprivileged)
665                        }
666                        _ => Err(BpfFilterStatus::Unsupported),
667                    };
668                }
669            } else {
670                // Negative xsk_fd is invalid for map update
671                return Err(BpfFilterStatus::Unsupported);
672            }
673        }
674
675        // 3. Load XDP program
676        let prog_fd = match bpf_prog_load_xdp(listen_port, map_fd) {
677            Ok(fd) => fd,
678            Err(e) => {
679                return match e.raw_os_error() {
680                    Some(libc::EPERM | libc::EACCES) => Err(BpfFilterStatus::FallbackUnprivileged),
681                    _ => Err(BpfFilterStatus::Unsupported),
682                };
683            }
684        };
685        let prog_guard = AutoCloseFd(prog_fd);
686
687        // 4. Attach XDP program to interface
688        if let Err(e) = bpf_attach_xdp(ifindex, prog_fd) {
689            return match e.raw_os_error() {
690                Some(libc::EPERM | libc::EACCES) => Err(BpfFilterStatus::FallbackUnprivileged),
691                _ => Err(BpfFilterStatus::Unsupported),
692            };
693        }
694
695        let map_fd = map_guard.into_raw();
696        let prog_fd = prog_guard.into_raw();
697        Ok(Self { map_fd, prog_fd })
698    }
699
700    /// Loads and attaches an XDP redirect filter to `ifname` for `listen_port` on a single queue.
701    pub fn load_and_attach(
702        ifname: &str,
703        listen_port: u16,
704        queue_id: u32,
705        xsk_fd: RawFd,
706    ) -> Result<Self, BpfFilterStatus> {
707        Self::load_multi_queue(ifname, listen_port, &[(queue_id, xsk_fd)])
708    }
709
710    /// Opportunistically attempts to attach the eBPF XDP redirect filter across multiple queues.
711    ///
712    /// Degrades fail-soft to `FallbackUnprivileged` or `Unsupported` without panicking.
713    pub fn attach_multi_queue(
714        ifname: &str,
715        listen_port: u16,
716        queue_fds: &[(u32, RawFd)],
717    ) -> BpfFilterStatus {
718        match Self::load_multi_queue(ifname, listen_port, queue_fds) {
719            Ok(filter) => {
720                let (map_fd, prog_fd) = filter.into_raw();
721                if map_fd >= 0 {
722                    // SAFETY: Closing unexported map descriptor; kernel keeps map reference alive via BPF prog.
723                    unsafe {
724                        libc::close(map_fd);
725                    }
726                }
727                BpfFilterStatus::Attached(prog_fd)
728            }
729            Err(status) => status,
730        }
731    }
732
733    /// Opportunistically attempts to attach the eBPF XDP redirect filter on a single queue.
734    ///
735    /// Degrades fail-soft to `FallbackUnprivileged` or `Unsupported` without panicking.
736    pub fn attach_opportunistic(
737        ifname: &str,
738        listen_port: u16,
739        queue_id: u32,
740        xsk_fd: RawFd,
741    ) -> BpfFilterStatus {
742        Self::attach_multi_queue(ifname, listen_port, &[(queue_id, xsk_fd)])
743    }
744}
745
746impl Drop for XdpRedirectFilter {
747    fn drop(&mut self) {
748        if self.map_fd >= 0 {
749            // SAFETY: self.map_fd is an open file descriptor owned by XdpRedirectFilter.
750            // Closing it releases the kernel BPF map resource.
751            unsafe {
752                libc::close(self.map_fd);
753            }
754            self.map_fd = -1;
755        }
756        if self.prog_fd >= 0 {
757            // SAFETY: self.prog_fd is an open file descriptor owned by XdpRedirectFilter.
758            // Closing it releases userspace reference to the BPF program.
759            unsafe {
760                libc::close(self.prog_fd);
761            }
762            self.prog_fd = -1;
763        }
764    }
765}
yip_io/bpf.rs

yip_io/
bpf.rs
