syd 3.58.0

//
// Syd: rock-solid application kernel
// src/t/uring.rs: Safe io_uring(7) interface for tests
//
// Copyright (c) 2026 Ali Polatel <alip@chesswob.org>
//
// SPDX-License-Identifier: GPL-3.0

//! Safe io_uring(7) interface for tests

#![allow(dead_code)]
#![allow(clippy::undocumented_unsafe_blocks)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::multiple_unsafe_ops_per_block)]
#![allow(clippy::cognitive_complexity)]
#![allow(clippy::disallowed_methods)]
#![allow(clippy::disallowed_types)]
#![allow(clippy::arithmetic_side_effects)]
#![allow(clippy::cast_possible_truncation)]
#![allow(clippy::cast_possible_wrap)]
#![allow(clippy::cast_sign_loss)]
#![allow(clippy::cast_lossless)]
#![allow(clippy::as_underscore)]
#![allow(clippy::decimal_literal_representation)]

use std::{
    mem::size_of,
    num::NonZeroUsize,
    os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd},
    ptr::NonNull,
    sync::atomic::{AtomicU32, Ordering},
};

use libc::{c_void, off_t};
use nix::{
    errno::Errno,
    sys::mman::{mmap, munmap, MapFlags, ProtFlags},
};

const IORING_OFF_SQ_RING: u64 = 0;
const IORING_OFF_CQ_RING: u64 = 0x0800_0000;
const IORING_OFF_SQES: u64 = 0x1000_0000;

const IORING_ENTER_GETEVENTS: u32 = 1;

const IORING_FEAT_SINGLE_MMAP: u32 = 1;

/// io_uring(7) submission queue opcodes
pub mod op {
    pub const NOP: u8 = 0;
    pub const READV: u8 = 1;
    pub const WRITEV: u8 = 2;
    pub const FSYNC: u8 = 3;
    pub const READ_FIXED: u8 = 4;
    pub const WRITE_FIXED: u8 = 5;
    pub const POLL_ADD: u8 = 6;
    pub const POLL_REMOVE: u8 = 7;
    pub const SYNC_FILE_RANGE: u8 = 8;
    pub const TIMEOUT: u8 = 11;
    pub const ACCEPT: u8 = 13;
    pub const CONNECT: u8 = 16;
    pub const FALLOCATE: u8 = 17;
    pub const OPENAT: u8 = 18;
    pub const CLOSE: u8 = 19;
    pub const STATX: u8 = 21;
    pub const READ: u8 = 22;
    pub const WRITE: u8 = 23;
    pub const FADVISE: u8 = 24;
    pub const MADVISE: u8 = 25;
    pub const SEND: u8 = 26;
    pub const RECV: u8 = 27;
    pub const OPENAT2: u8 = 28;
    pub const EPOLL_CTL: u8 = 29;
    pub const SPLICE: u8 = 30;
    pub const TEE: u8 = 33;
    pub const SHUTDOWN: u8 = 34;
    pub const RENAMEAT: u8 = 35;
    pub const UNLINKAT: u8 = 36;
    pub const MKDIRAT: u8 = 37;
    pub const SYMLINKAT: u8 = 38;
    pub const LINKAT: u8 = 39;
    pub const SOCKET: u8 = 45;
}

/// io_uring(7) submission queue entry flags
pub mod flag {
    pub const FIXED_FILE: u8 = 1 << 0;
    pub const ASYNC: u8 = 1 << 4;
}

#[repr(C)]
#[derive(Clone, Copy, Default)]
struct IoUringSqe {
    opcode: u8,
    flags: u8,
    ioprio: u16,
    fd: i32,
    off: u64,
    addr: u64,
    len: u32,
    op_flags: u32,
    user_data: u64,
    buf_index: u16,
    personality: u16,
    splice_fd_in: u32,
    addr3: u64,
    pad2: u64,
}

#[repr(C)]
#[derive(Clone, Copy, Default)]
struct IoUringCqe {
    user_data: u64,
    res: i32,
    flags: u32,
}

#[repr(C)]
#[derive(Clone, Copy, Default)]
struct IoSqringOffsets {
    head: u32,
    tail: u32,
    ring_mask: u32,
    ring_entries: u32,
    flags: u32,
    dropped: u32,
    array: u32,
    resv1: u32,
    user_addr: u64,
}

#[repr(C)]
#[derive(Clone, Copy, Default)]
struct IoCqringOffsets {
    head: u32,
    tail: u32,
    ring_mask: u32,
    ring_entries: u32,
    overflow: u32,
    cqes: u32,
    flags: u32,
    resv1: u32,
    user_addr: u64,
}

#[repr(C)]
#[derive(Clone, Copy, Default)]
struct IoUringParams {
    sq_entries: u32,
    cq_entries: u32,
    flags: u32,
    sq_thread_cpu: u32,
    sq_thread_idle: u32,
    features: u32,
    wq_fd: u32,
    resv: [u32; 3],
    sq_off: IoSqringOffsets,
    cq_off: IoCqringOffsets,
}

const _: () = assert!(size_of::<IoUringSqe>() == 64);
const _: () = assert!(size_of::<IoUringCqe>() == 16);
const _: () = assert!(size_of::<IoUringParams>() == 120);

/// Submission queue entry builder.
pub struct Sqe {
    raw: IoUringSqe,
}

impl Sqe {
    fn rw(opcode: u8, fd: i32, addr: u64, len: u32, off: u64) -> Sqe {
        Sqe {
            raw: IoUringSqe {
                opcode,
                fd,
                addr,
                len,
                off,
                ..Default::default()
            },
        }
    }

    /// No-op submission.
    pub fn nop() -> Sqe {
        Self::rw(op::NOP, -1, 0, 0, 0)
    }

    /// Read from `fd` into `buf` of `len` bytes at offset zero.
    pub fn read(fd: RawFd, buf: *mut u8, len: u32) -> Sqe {
        Self::rw(op::READ, fd, buf as usize as u64, len, 0)
    }

    /// Write to `fd` from `buf` of `len` bytes at offset zero.
    pub fn write(fd: RawFd, buf: *const u8, len: u32) -> Sqe {
        Self::rw(op::WRITE, fd, buf as usize as u64, len, 0)
    }

    /// Fsync of `fd`.
    pub fn fsync(fd: RawFd) -> Sqe {
        Self::rw(op::FSYNC, fd, 0, 0, 0)
    }

    /// Openat of `path` relative to `dfd` with flags and mode zero.
    pub fn openat(dfd: RawFd, path: *const libc::c_char) -> Sqe {
        Self::rw(op::OPENAT, dfd, path as usize as u64, 0, 0)
    }

    /// Create socket.
    pub fn socket(domain: i32, ty: i32, protocol: i32) -> Sqe {
        Self::rw(op::SOCKET, domain, 0, protocol as u32, ty as u64)
    }

    /// Set submission queue entry flags.
    pub fn flags(mut self, flags: u8) -> Sqe {
        self.raw.flags |= flags;
        self
    }
}

// Ring region
//
// Allocated with mmap(2).
// Unmapped with munmap(2) on Drop.
struct Region {
    ptr: NonNull<c_void>,
    len: usize,
}

/// Safe io_uring(7) interface
pub struct IoUring {
    fd: OwnedFd,
    sq_ring: Region,
    cq_ring: Region,
    sqes_ring: Region,
    sq_khead: *mut u32,
    sq_ktail: *mut u32,
    sq_array: *mut u32,
    sq_ring_mask: u32,
    sq_ring_entries: u32,
    sqes: *mut IoUringSqe,
    cq_khead: *mut u32,
    cq_ktail: *mut u32,
    cq_ring_mask: u32,
    cqes: *mut IoUringCqe,
    sq_tail: u32,
    to_submit: u32,
}

fn ring_load(ptr: *mut u32, order: Ordering) -> u32 {
    // SAFETY: ptr is an aligned u32 within a mmap'd ring.
    unsafe { AtomicU32::from_ptr(ptr).load(order) }
}

fn ring_store(ptr: *mut u32, val: u32, order: Ordering) {
    // SAFETY: ptr is an aligned u32 within a mmap'd ring.
    unsafe { AtomicU32::from_ptr(ptr).store(val, order) }
}

impl IoUring {
    /// Set up a ring with room for `entries` submissions.
    pub fn new(entries: u32) -> Result<IoUring, Errno> {
        let mut params = IoUringParams::default();

        // SAFETY: Params points to a valid, zeroed io_uring_params.
        let fd = Errno::result(unsafe {
            libc::syscall(
                libc::SYS_io_uring_setup,
                entries,
                std::ptr::addr_of_mut!(params).cast::<c_void>(),
            )
        })
        .map(|fd| {
            // SAFETY: io_uring_setup(2) returns a valid FD on success.
            unsafe { OwnedFd::from_raw_fd(fd as RawFd) }
        })?;

        let single = params.features & IORING_FEAT_SINGLE_MMAP != 0;
        let mut sq_len =
            params.sq_off.array as usize + params.sq_entries as usize * size_of::<u32>();
        let mut cq_len =
            params.cq_off.cqes as usize + params.cq_entries as usize * size_of::<IoUringCqe>();
        if single {
            let max = sq_len.max(cq_len);
            sq_len = max;
            cq_len = max;
        }
        let sqes_len = params.sq_entries as usize * size_of::<IoUringSqe>();

        let sq_ptr = map(&fd, sq_len, IORING_OFF_SQ_RING)?;
        let cq_ptr = if single {
            sq_ptr
        } else {
            map(&fd, cq_len, IORING_OFF_CQ_RING)?
        };
        let sqes_ptr = map(&fd, sqes_len, IORING_OFF_SQES)?;

        let sq_addr = sq_ptr.as_ptr() as usize;
        let cq_addr = cq_ptr.as_ptr() as usize;
        let sqes_addr = sqes_ptr.as_ptr() as usize;
        let u32_at = |base: usize, off: u32| (base + off as usize) as *mut u32;

        // SAFETY: Ring offsets are within bounds.
        let ring = unsafe {
            IoUring {
                sq_khead: u32_at(sq_addr, params.sq_off.head),
                sq_ktail: u32_at(sq_addr, params.sq_off.tail),
                sq_array: u32_at(sq_addr, params.sq_off.array),
                sq_ring_mask: *u32_at(sq_addr, params.sq_off.ring_mask),
                sq_ring_entries: *u32_at(sq_addr, params.sq_off.ring_entries),
                sqes: sqes_addr as *mut IoUringSqe,
                cq_khead: u32_at(cq_addr, params.cq_off.head),
                cq_ktail: u32_at(cq_addr, params.cq_off.tail),
                cq_ring_mask: *u32_at(cq_addr, params.cq_off.ring_mask),
                cqes: (cq_addr + params.cq_off.cqes as usize) as *mut IoUringCqe,
                fd,
                sq_ring: Region {
                    ptr: sq_ptr,
                    len: sq_len,
                },
                cq_ring: Region {
                    ptr: cq_ptr,
                    len: cq_len,
                },
                sqes_ring: Region {
                    ptr: sqes_ptr,
                    len: sqes_len,
                },
                sq_tail: 0,
                to_submit: 0,
            }
        };

        Ok(ring)
    }

    /// Queue one submission queue entry.
    pub fn push(&mut self, sqe: &Sqe) -> Result<(), Errno> {
        let khead = ring_load(self.sq_khead, Ordering::Acquire);
        if self.sq_tail.wrapping_sub(khead) >= self.sq_ring_entries {
            return Err(Errno::EAGAIN);
        }

        let index = self.sq_tail & self.sq_ring_mask;

        // SAFETY: Index is masked into sqes and array regions.
        unsafe {
            *self.sqes.add(index as usize) = sqe.raw;
            *self.sq_array.add(index as usize) = index;
        }

        self.sq_tail = self.sq_tail.wrapping_add(1);
        ring_store(self.sq_ktail, self.sq_tail, Ordering::Release);
        self.to_submit = self.to_submit.wrapping_add(1);

        Ok(())
    }

    /// Submit queued entries without waiting.
    pub fn submit(&mut self) -> Result<(), Errno> {
        self.enter(0)
    }

    /// Submit queued entries and wait for `want` completions.
    pub fn submit_and_wait(&mut self, want: u32) -> Result<(), Errno> {
        self.enter(want)
    }

    fn enter(&mut self, want: u32) -> Result<(), Errno> {
        let flags: u32 = if want > 0 { IORING_ENTER_GETEVENTS } else { 0 };

        // SAFETY:
        // 1. fd is a valid ring
        // 2. Signal mask is null with size zero.
        Errno::result(unsafe {
            libc::syscall(
                libc::SYS_io_uring_enter,
                self.fd.as_raw_fd(),
                self.to_submit,
                want,
                flags,
                std::ptr::null::<c_void>(),
                0_usize,
            )
        })?;

        self.to_submit = 0;

        Ok(())
    }

    /// Pop one completion result, if any is available.
    pub fn completion(&mut self) -> Option<i32> {
        let head = ring_load(self.cq_khead, Ordering::Acquire);
        let tail = ring_load(self.cq_ktail, Ordering::Acquire);

        if head == tail {
            return None;
        }

        let index = head & self.cq_ring_mask;

        // SAFETY: Index is masked into cqes region and head != tail.
        let res = unsafe { (*self.cqes.add(index as usize)).res };

        ring_store(self.cq_khead, head.wrapping_add(1), Ordering::Release);

        Some(res)
    }
}

impl Drop for IoUring {
    fn drop(&mut self) {
        // SAFETY: These regions are allocated by new() using mmap(2).
        unsafe {
            let _ = munmap(self.sq_ring.ptr, self.sq_ring.len);
            if self.cq_ring.ptr != self.sq_ring.ptr {
                let _ = munmap(self.cq_ring.ptr, self.cq_ring.len);
            }
            let _ = munmap(self.sqes_ring.ptr, self.sqes_ring.len);
        }
    }
}

fn map(fd: &OwnedFd, len: usize, offset: u64) -> Result<NonNull<c_void>, Errno> {
    let len = NonZeroUsize::new(len).ok_or(Errno::EINVAL)?;

    // SAFETY:
    // 1. fd is a valid io_uring(7) ring.
    // 2. Offset is a ring pseudo offset.
    unsafe {
        mmap(
            None,
            len,
            ProtFlags::PROT_READ | ProtFlags::PROT_WRITE,
            MapFlags::MAP_SHARED | MapFlags::MAP_POPULATE,
            fd,
            offset as off_t,
        )
    }
}