Skip to main content

io_uring/
submit.rs

1use std::os::unix::io::{AsRawFd, RawFd};
2use std::sync::atomic;
3use std::{io, mem, ptr};
4
5use crate::register::{execute, Probe, RegisterRing};
6use crate::sys;
7use crate::types::{CancelBuilder, CloneBuffersFlags, Napi, Timespec};
8use crate::util::{cast_ptr, OwnedFd};
9use crate::Parameters;
10use bitflags::bitflags;
11
12use crate::register::Restriction;
13
14use crate::types;
15
16bitflags!(
17    /// See man page for complete description:
18    /// https://man7.org/linux/man-pages/man2/io_uring_enter.2.html
19    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
20    pub struct EnterFlags: u32 {
21        /// Wait for at least `min_complete` events to complete.
22        const GETEVENTS = sys::IORING_ENTER_GETEVENTS;
23
24        /// If the kernel thread is sleeping, wake it up.
25        const SQ_WAKEUP = sys::IORING_ENTER_SQ_WAKEUP;
26
27        /// Wait for at least one submission queue entry to be available.
28        const SQ_WAIT = sys::IORING_ENTER_SQ_WAIT;
29
30        /// Use the extended argument structure.
31        const EXT_ARG = sys::IORING_ENTER_EXT_ARG;
32
33        /// Submit using registered submission queue ring.
34        const REGISTERED_RING = sys::IORING_ENTER_REGISTERED_RING;
35
36        /// Timeout argument interpreted as absolute time.
37        const ABS_TIMER = sys::IORING_ENTER_ABS_TIMER;
38
39        /// Arg is offset into an area of wait regions previously registered.
40        const EXT_ARG_REG = sys::IORING_ENTER_EXT_ARG_REG;
41
42        /// Don't mark waiting task as being in iowait in certain cases.
43        const NO_IOWAIT = sys::IORING_ENTER_NO_IOWAIT;
44    }
45);
46
47/// Interface for submitting submission queue events in an io_uring instance to the kernel for
48/// executing and registering files or buffers with the instance.
49///
50/// io_uring supports both directly performing I/O on buffers and file descriptors and registering
51/// them beforehand. Registering is slow, but it makes performing the actual I/O much faster.
52pub struct Submitter<'a> {
53    fd: &'a OwnedFd,
54    params: &'a Parameters,
55    enter_ring_fd: i32,
56
57    sq_head: *const atomic::AtomicU32,
58    sq_tail: *const atomic::AtomicU32,
59    sq_flags: *const atomic::AtomicU32,
60}
61
62impl<'a> Submitter<'a> {
63    #[inline]
64    pub(crate) const fn new(
65        fd: &'a OwnedFd,
66        params: &'a Parameters,
67        sq_head: *const atomic::AtomicU32,
68        sq_tail: *const atomic::AtomicU32,
69        sq_flags: *const atomic::AtomicU32,
70    ) -> Submitter<'a> {
71        Submitter {
72            fd,
73            params,
74            enter_ring_fd: -1,
75            sq_head,
76            sq_tail,
77            sq_flags,
78        }
79    }
80
81    #[inline]
82    fn sq_len(&self) -> usize {
83        unsafe {
84            let head = (*self.sq_head).load(atomic::Ordering::Acquire);
85            let tail = (*self.sq_tail).load(atomic::Ordering::Acquire);
86
87            tail.wrapping_sub(head) as usize
88        }
89    }
90
91    /// Whether the kernel thread has gone to sleep because it waited for too long without
92    /// submission queue entries.
93    #[inline]
94    fn sq_need_wakeup(&self) -> bool {
95        unsafe {
96            (*self.sq_flags).load(atomic::Ordering::Relaxed) & sys::IORING_SQ_NEED_WAKEUP != 0
97        }
98    }
99
100    /// Load flags published by the kernel in the submission queue ring.
101    #[inline]
102    fn load_sq_flags(&self) -> u32 {
103        unsafe { (*self.sq_flags).load(atomic::Ordering::Acquire) }
104    }
105
106    #[inline]
107    fn execute_register(
108        &self,
109        opcode: libc::c_uint,
110        arg: *const libc::c_void,
111        len: libc::c_uint,
112    ) -> io::Result<i32> {
113        execute(self.register_ring(), opcode, arg, len)
114    }
115
116    #[inline]
117    fn register_ring(&self) -> RegisterRing {
118        let enter_ring_fd = self.enter_ring_fd;
119        if enter_ring_fd >= 0 && self.params.is_feature_reg_reg_ring() {
120            RegisterRing::RegisteredIndex(enter_ring_fd)
121        } else {
122            RegisterRing::RawFd(self.fd.as_raw_fd())
123        }
124    }
125
126    /// Initiate and/or complete asynchronous I/O. This is a low-level wrapper around
127    /// `io_uring_enter` - see `man io_uring_enter` (or [its online
128    /// version](https://manpages.debian.org/unstable/liburing-dev/io_uring_enter.2.en.html) for
129    /// more details.
130    ///
131    /// You will probably want to use a more high-level API such as
132    /// [`submit`](Self::submit) or [`submit_and_wait`](Self::submit_and_wait).
133    ///
134    /// # Safety
135    ///
136    /// This provides a raw interface so developer must ensure that parameters are correct.
137    pub unsafe fn enter<T: Sized>(
138        &self,
139        to_submit: u32,
140        min_complete: u32,
141        flag: u32,
142        arg: Option<&T>,
143    ) -> io::Result<usize> {
144        let arg = arg
145            .map(|arg| cast_ptr(arg).cast())
146            .unwrap_or_else(ptr::null);
147        let size = mem::size_of::<T>();
148
149        // If a ring fd has been registered with [`register_ring_fd`](Self::register_ring_fd),
150        // `enter_ring_fd` holds its index (otherwise `-1`); pass it together with
151        // `IORING_ENTER_REGISTERED_RING` instead of the raw file descriptor to avoid the per-call
152        // fd lookup in the kernel.
153        let enter_ring_fd = self.enter_ring_fd;
154        let (fd, flag) = if enter_ring_fd >= 0 {
155            (enter_ring_fd, flag | sys::IORING_ENTER_REGISTERED_RING)
156        } else {
157            (self.fd.as_raw_fd(), flag)
158        };
159
160        sys::io_uring_enter(fd, to_submit, min_complete, flag, arg, size).map(|res| res as _)
161    }
162
163    /// Submit all queued submission queue events to the kernel.
164    #[inline]
165    pub fn submit(&self) -> io::Result<usize> {
166        self.submit_and_wait(0)
167    }
168
169    /// Submit all queued submission queue events to the kernel and wait for at least `want`
170    /// completion events to complete.
171    pub fn submit_and_wait(&self, want: usize) -> io::Result<usize> {
172        let len = self.sq_len();
173        let mut flags = EnterFlags::empty();
174
175        // This logic suffers from the fact the sq_cq_overflow and sq_need_wakeup
176        // each cause an atomic load of the same variable, self.sq_flags.
177        // In the hottest paths, when a server is running with sqpoll,
178        // this is going to be hit twice, when once would be sufficient.
179        // However, consider that the `SeqCst` barrier required for interpreting
180        // the IORING_ENTER_SQ_WAKEUP bit is required in all paths where sqpoll
181        // is setup when consolidating the reads.
182
183        let sq_flags = self.load_sq_flags();
184        let sq_cq_overflow = sq_flags & sys::IORING_SQ_CQ_OVERFLOW != 0;
185        let sq_taskrun = sq_flags & sys::IORING_SQ_TASKRUN != 0;
186
187        // When IORING_FEAT_NODROP is enabled and CQ overflows, the kernel buffers
188        // completion events internally but doesn't automatically flush them when
189        // CQ space becomes available. We must explicitly call io_uring_enter()
190        // to flush these buffered events, even with SQPOLL enabled.
191        //
192        // Without this, completions remain stuck in kernel's internal buffer
193        // after draining CQ, causing missing completion notifications.
194        let need_syscall = (sq_cq_overflow && self.params.is_feature_nodrop()) || sq_taskrun;
195
196        // Deferred task work is only run by an enter carrying GETEVENTS.
197        if want > 0 || self.params.is_setup_iopoll() || sq_cq_overflow || sq_taskrun {
198            flags.insert(EnterFlags::GETEVENTS);
199        }
200
201        if self.params.is_setup_sqpoll() {
202            // See discussion in [`SubmissionQueue::need_wakeup`].
203            atomic::fence(atomic::Ordering::SeqCst);
204            if self.sq_need_wakeup() {
205                flags.insert(EnterFlags::SQ_WAKEUP);
206            } else if want == 0 && !need_syscall {
207                // The kernel thread is polling and hasn't fallen asleep, so we don't need to tell
208                // it to process events or wake it up
209
210                // However, if the CQ ring is overflown, we need to tell the kernel to process events
211                // by calling io_uring_enter with the IORING_ENTER_GETEVENTS flag.
212                return Ok(len);
213            }
214        }
215
216        unsafe { self.enter::<libc::sigset_t>(len as _, want as _, flags.bits(), None) }
217    }
218
219    /// Submit all queued submission queue events to the kernel and wait for at least `want`
220    /// completion events to complete with additional options
221    ///
222    /// You can specify a set of signals to mask and a timeout for operation, see
223    /// [`SubmitArgs`](types::SubmitArgs) for more details
224    pub fn submit_with_args(
225        &self,
226        want: usize,
227        args: &types::SubmitArgs<'_, '_>,
228    ) -> io::Result<usize> {
229        self.submit_with_args_and_flags(want, args, EnterFlags::empty())
230    }
231
232    /// Like [`submit_with_args`](Self::submit_with_args), with extra `flags` for the wait.
233    ///
234    /// Only these flags are accepted; any other fails with `EINVAL`, without entering the kernel:
235    ///
236    /// - [`EnterFlags::ABS_TIMER`]: the timeout in `args` is an absolute time on the ring's clock
237    ///   (`CLOCK_MONOTONIC` unless registered otherwise), not a duration. Available since kernel
238    ///   6.12.
239    /// - [`EnterFlags::NO_IOWAIT`]: the wait is not accounted as iowait. See
240    ///   [`Parameters::is_feature_no_iowait`](crate::Parameters::is_feature_no_iowait).
241    ///
242    /// A kernel that doesn't know a flag fails with `EINVAL`.
243    pub fn submit_with_args_and_flags(
244        &self,
245        want: usize,
246        args: &types::SubmitArgs<'_, '_>,
247        flags: EnterFlags,
248    ) -> io::Result<usize> {
249        if !(EnterFlags::ABS_TIMER | EnterFlags::NO_IOWAIT).contains(flags) {
250            return Err(io::Error::from_raw_os_error(libc::EINVAL));
251        }
252
253        let len = self.sq_len();
254        let mut flags = flags | EnterFlags::EXT_ARG;
255
256        let sq_flags = self.load_sq_flags();
257        let sq_cq_overflow = sq_flags & sys::IORING_SQ_CQ_OVERFLOW != 0;
258        let sq_taskrun = sq_flags & sys::IORING_SQ_TASKRUN != 0;
259        let need_syscall = (sq_cq_overflow && self.params.is_feature_nodrop()) || sq_taskrun;
260
261        // Deferred task work is only run by an enter carrying GETEVENTS.
262        if want > 0 || self.params.is_setup_iopoll() || sq_cq_overflow || sq_taskrun {
263            flags.insert(EnterFlags::GETEVENTS);
264        }
265
266        if self.params.is_setup_sqpoll() {
267            // See discussion in [`SubmissionQueue::need_wakeup`].
268            atomic::fence(atomic::Ordering::SeqCst);
269            if self.sq_need_wakeup() {
270                flags.insert(EnterFlags::SQ_WAKEUP);
271            } else if want == 0 && !need_syscall {
272                // The kernel thread is polling and hasn't fallen asleep, so we don't need to tell
273                // it to process events or wake it up
274                return Ok(len);
275            }
276        }
277
278        unsafe { self.enter(len as _, want as _, flags.bits(), Some(args)) }
279    }
280
281    /// Wait for the submission queue to have free entries.
282    pub fn squeue_wait(&self) -> io::Result<usize> {
283        unsafe { self.enter::<libc::sigset_t>(0, 0, EnterFlags::SQ_WAIT.bits(), None) }
284    }
285
286    /// Register in-memory fixed buffers for I/O with the kernel. You can use these buffers with the
287    /// [`ReadFixed`](crate::opcode::ReadFixed) and [`WriteFixed`](crate::opcode::WriteFixed)
288    /// operations.
289    ///
290    /// # Safety
291    ///
292    /// Developers must ensure that the `iov_base` and `iov_len` values are valid and will
293    /// be valid until buffers are unregistered or the ring destroyed, otherwise undefined
294    /// behaviour may occur.
295    pub unsafe fn register_buffers(&self, bufs: &[libc::iovec]) -> io::Result<()> {
296        self.execute_register(
297            sys::IORING_REGISTER_BUFFERS,
298            bufs.as_ptr().cast(),
299            bufs.len() as _,
300        )
301        .map(drop)
302    }
303
304    /// Update a range of fixed buffers starting at `offset`.
305    ///
306    /// This is required to use buffers registered using
307    /// [`register_buffers_sparse`](Self::register_buffers_sparse),
308    /// although it can be also be used with [`register_buffers`](Self::register_buffers).
309    ///
310    /// See [`register_buffers2`](Self::register_buffers2)
311    /// for more information about resource tagging.
312    ///
313    /// Available since Linux 5.13.
314    ///
315    /// # Safety
316    ///
317    /// Developers must ensure that the `iov_base` and `iov_len` values are valid and will
318    /// be valid until buffers are unregistered or the ring destroyed, otherwise undefined
319    /// behaviour may occur.
320    pub unsafe fn register_buffers_update(
321        &self,
322        offset: u32,
323        bufs: &[libc::iovec],
324        tags: Option<&[u64]>,
325    ) -> io::Result<()> {
326        let nr = tags
327            .as_ref()
328            .map_or(bufs.len(), |tags| bufs.len().min(tags.len()));
329
330        let rr = sys::io_uring_rsrc_update2 {
331            nr: nr as _,
332            data: bufs.as_ptr() as _,
333            tags: tags.map(|tags| tags.as_ptr() as _).unwrap_or(0),
334            offset,
335            ..Default::default()
336        };
337
338        self.execute_register(
339            sys::IORING_REGISTER_BUFFERS_UPDATE,
340            cast_ptr::<sys::io_uring_rsrc_update2>(&rr).cast(),
341            std::mem::size_of::<sys::io_uring_rsrc_update2>() as _,
342        )
343        .map(drop)
344    }
345
346    /// Variant of [`register_buffers`](Self::register_buffers)
347    /// with resource tagging.
348    ///
349    /// `tags` should be the same length as `bufs` and contain the
350    /// tag value corresponding to the buffer at the same index.
351    ///
352    /// If a tag is zero, then tagging for this particular resource
353    /// (a buffer in this case) is disabled. Otherwise, after the
354    /// resource had been unregistered and it's not used anymore,
355    /// a CQE will be posted with `user_data` set to the specified
356    /// tag and all other fields zeroed.
357    ///
358    /// Available since Linux 5.13.
359    ///
360    /// # Safety
361    ///
362    /// Developers must ensure that the `iov_base` and `iov_len` values are valid and will
363    /// be valid until buffers are unregistered or the ring destroyed, otherwise undefined
364    /// behaviour may occur.
365    pub unsafe fn register_buffers2(&self, bufs: &[libc::iovec], tags: &[u64]) -> io::Result<()> {
366        let rr = sys::io_uring_rsrc_register {
367            nr: bufs.len().min(tags.len()) as _,
368            data: bufs.as_ptr() as _,
369            tags: tags.as_ptr() as _,
370            ..Default::default()
371        };
372        self.execute_register(
373            sys::IORING_REGISTER_BUFFERS2,
374            cast_ptr::<sys::io_uring_rsrc_register>(&rr).cast(),
375            std::mem::size_of::<sys::io_uring_rsrc_register>() as _,
376        )
377        .map(drop)
378    }
379
380    /// Registers an empty table of nr fixed buffers buffers.
381    ///
382    /// These must be updated before use, using eg.
383    /// [`register_buffers_update`](Self::register_buffers_update).
384    ///
385    /// See [`register_buffers`](Self::register_buffers)
386    /// for more information about fixed buffers.
387    ///
388    /// Available since Linux 5.13.
389    pub fn register_buffers_sparse(&self, nr: u32) -> io::Result<()> {
390        let rr = sys::io_uring_rsrc_register {
391            nr,
392            flags: sys::IORING_RSRC_REGISTER_SPARSE,
393            ..Default::default()
394        };
395        self.execute_register(
396            sys::IORING_REGISTER_BUFFERS2,
397            cast_ptr::<sys::io_uring_rsrc_register>(&rr).cast(),
398            std::mem::size_of::<sys::io_uring_rsrc_register>() as _,
399        )
400        .map(drop)
401    }
402
403    /// Clone the entire registered buffer table from another ring into this one.
404    ///
405    /// `src_fd` is the raw file descriptor of the source `io_uring`. The source's
406    /// buffers are shared with this ring rather than copied, so a single physical
407    /// registration can back many rings without re-pinning the pages in the kernel.
408    ///
409    /// This ring's buffer table must be empty. To clone into a non-empty table or
410    /// to copy a sub-range, use
411    /// [`register_buffers_clone_offset`](Self::register_buffers_clone_offset).
412    ///
413    /// Available since Linux 6.12.
414    pub fn register_buffers_clone(&self, src_fd: RawFd) -> io::Result<()> {
415        self.register_buffers_clone_offset(src_fd, 0, 0, 0, CloneBuffersFlags::empty())
416    }
417
418    /// Clone a range of the registered buffer table from another ring into this one.
419    ///
420    /// `src_fd` is the raw file descriptor of the source `io_uring`. `nr` buffers
421    /// starting at `src_off` in the source table are installed starting at `dst_off`
422    /// in this ring's table. A `nr` of `0` clones the source's entire table.
423    ///
424    /// See [`CloneBuffersFlags`] for replacing an existing destination range or
425    /// treating `src_fd` as a registered ring descriptor.
426    ///
427    /// Available since Linux 6.12.
428    pub fn register_buffers_clone_offset(
429        &self,
430        src_fd: RawFd,
431        src_off: u32,
432        dst_off: u32,
433        nr: u32,
434        flags: CloneBuffersFlags,
435    ) -> io::Result<()> {
436        let arg = sys::io_uring_clone_buffers {
437            src_fd: src_fd as _,
438            flags: flags.bits(),
439            src_off,
440            dst_off,
441            nr,
442            ..Default::default()
443        };
444        execute(
445            RegisterRing::RawFd(self.fd.as_raw_fd()),
446            sys::IORING_REGISTER_CLONE_BUFFERS,
447            cast_ptr::<sys::io_uring_clone_buffers>(&arg).cast(),
448            // This opcode takes a single struct; the kernel requires nr_args == 1.
449            1,
450        )
451        .map(drop)
452    }
453
454    /// Registers an empty file table of nr_files number of file descriptors. The sparse variant is
455    /// available in kernels 5.19 and later.
456    ///
457    /// Registering a file table is a prerequisite for using any request that
458    /// uses direct descriptors.
459    pub fn register_files_sparse(&self, nr: u32) -> io::Result<()> {
460        let rr = sys::io_uring_rsrc_register {
461            nr,
462            flags: sys::IORING_RSRC_REGISTER_SPARSE,
463            resv2: 0,
464            data: 0,
465            tags: 0,
466        };
467        self.execute_register(
468            sys::IORING_REGISTER_FILES2,
469            cast_ptr::<sys::io_uring_rsrc_register>(&rr).cast(),
470            mem::size_of::<sys::io_uring_rsrc_register>() as _,
471        )
472        .map(drop)
473    }
474
475    /// Register files for I/O. You can use the registered files with
476    /// [`Fixed`](crate::types::Fixed).
477    ///
478    /// Each fd may be -1, in which case it is considered "sparse", and can be filled in later with
479    /// [`register_files_update`](Self::register_files_update).
480    ///
481    /// Note that this will wait for the ring to idle; it will only return once all active requests
482    /// are complete. Use [`register_files_update`](Self::register_files_update) to avoid this.
483    pub fn register_files(&self, fds: &[RawFd]) -> io::Result<()> {
484        self.execute_register(
485            sys::IORING_REGISTER_FILES,
486            fds.as_ptr().cast(),
487            fds.len() as _,
488        )
489        .map(drop)
490    }
491
492    /// This operation replaces existing files in the registered file set with new ones,
493    /// either turning a sparse entry (one where fd is equal to -1) into a real one, removing an existing entry (new one is set to -1),
494    /// or replacing an existing entry with a new existing entry. The `offset` parameter specifies
495    /// the offset into the list of registered files at which to start updating files.
496    ///
497    /// You can also perform this asynchronously with the
498    /// [`FilesUpdate`](crate::opcode::FilesUpdate) opcode.
499    pub fn register_files_update(&self, offset: u32, fds: &[RawFd]) -> io::Result<usize> {
500        let fu = sys::io_uring_files_update {
501            offset,
502            resv: 0,
503            fds: fds.as_ptr() as _,
504        };
505        let ret = self.execute_register(
506            sys::IORING_REGISTER_FILES_UPDATE,
507            cast_ptr::<sys::io_uring_files_update>(&fu).cast(),
508            fds.len() as _,
509        )?;
510        Ok(ret as _)
511    }
512
513    /// Register an eventfd created by [`eventfd`](libc::eventfd) with the io_uring instance.
514    pub fn register_eventfd(&self, eventfd: RawFd) -> io::Result<()> {
515        self.execute_register(
516            sys::IORING_REGISTER_EVENTFD,
517            cast_ptr::<RawFd>(&eventfd).cast(),
518            1,
519        )
520        .map(drop)
521    }
522
523    /// This works just like [`register_eventfd`](Self::register_eventfd), except notifications are
524    /// only posted for events that complete in an async manner, so requests that complete
525    /// immediately will not cause a notification.
526    pub fn register_eventfd_async(&self, eventfd: RawFd) -> io::Result<()> {
527        self.execute_register(
528            sys::IORING_REGISTER_EVENTFD_ASYNC,
529            cast_ptr::<RawFd>(&eventfd).cast(),
530            1,
531        )
532        .map(drop)
533    }
534
535    /// Fill in the given [`Probe`] with information about the opcodes supported by io_uring on the
536    /// running kernel.
537    ///
538    /// # Examples
539    ///
540    // This is marked no_run as it is only available from Linux 5.6+, however the latest Ubuntu (on
541    // which CI runs) only has Linux 5.4.
542    /// ```no_run
543    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
544    /// let io_uring = io_uring::IoUring::new(1)?;
545    /// let mut probe = io_uring::Probe::new();
546    /// io_uring.submitter().register_probe(&mut probe)?;
547    ///
548    /// if probe.is_supported(io_uring::opcode::Read::CODE) {
549    ///     println!("Reading is supported!");
550    /// }
551    /// # Ok(())
552    /// # }
553    /// ```
554    pub fn register_probe(&self, probe: &mut Probe) -> io::Result<()> {
555        self.execute_register(
556            sys::IORING_REGISTER_PROBE,
557            probe.as_mut_ptr() as *const _,
558            Probe::COUNT as _,
559        )
560        .map(drop)
561    }
562
563    /// Register credentials of the running application with io_uring, and get an id associated with
564    /// these credentials. This ID can then be [passed](crate::squeue::Entry::personality) into
565    /// submission queue entries to issue the request with this process' credentials.
566    ///
567    /// By default, if [`Parameters::is_feature_cur_personality`] is set then requests will use the
568    /// credentials of the task that called [`Submitter::enter`], otherwise they will use the
569    /// credentials of the task that originally registered the io_uring.
570    ///
571    /// [`Parameters::is_feature_cur_personality`]: crate::Parameters::is_feature_cur_personality
572    pub fn register_personality(&self) -> io::Result<u16> {
573        let id = self.execute_register(sys::IORING_REGISTER_PERSONALITY, ptr::null(), 0)?;
574        Ok(id as u16)
575    }
576
577    /// Unregister all previously registered buffers.
578    ///
579    /// You do not need to explicitly call this before dropping the [`IoUring`](crate::IoUring), as
580    /// it will be cleaned up by the kernel automatically.
581    ///
582    /// Available since Linux 5.1.
583    pub fn unregister_buffers(&self) -> io::Result<()> {
584        self.execute_register(sys::IORING_UNREGISTER_BUFFERS, ptr::null(), 0)
585            .map(drop)
586    }
587
588    /// Unregister all previously registered files.
589    ///
590    /// You do not need to explicitly call this before dropping the [`IoUring`](crate::IoUring), as
591    /// it will be cleaned up by the kernel automatically.
592    pub fn unregister_files(&self) -> io::Result<()> {
593        self.execute_register(sys::IORING_UNREGISTER_FILES, ptr::null(), 0)
594            .map(drop)
595    }
596
597    /// Unregister an eventfd file descriptor to stop notifications.
598    pub fn unregister_eventfd(&self) -> io::Result<()> {
599        self.execute_register(sys::IORING_UNREGISTER_EVENTFD, ptr::null(), 0)
600            .map(drop)
601    }
602
603    /// Unregister a previously registered personality.
604    pub fn unregister_personality(&self, personality: u16) -> io::Result<()> {
605        self.execute_register(
606            sys::IORING_UNREGISTER_PERSONALITY,
607            ptr::null(),
608            personality as _,
609        )
610        .map(drop)
611    }
612
613    /// Permanently install a feature allowlist. Once this has been called, attempting to perform
614    /// an operation not on the allowlist will fail with `-EACCES`.
615    ///
616    /// This can only be called once, to prevent untrusted code from removing restrictions.
617    pub fn register_restrictions(&self, res: &mut [Restriction]) -> io::Result<()> {
618        self.execute_register(
619            sys::IORING_REGISTER_RESTRICTIONS,
620            res.as_mut_ptr().cast(),
621            res.len() as _,
622        )
623        .map(drop)
624    }
625
626    /// Enable the rings of the io_uring instance if they have been disabled with
627    /// [`setup_r_disabled`](crate::Builder::setup_r_disabled).
628    pub fn register_enable_rings(&self) -> io::Result<()> {
629        self.execute_register(sys::IORING_REGISTER_ENABLE_RINGS, ptr::null(), 0)
630            .map(drop)
631    }
632
633    /// Tell io_uring on what CPUs the async workers can run. By default, async workers
634    /// created by io_uring will inherit the CPU mask of its parent. This is usually
635    /// all the CPUs in the system, unless the parent is being run with a limited set.
636    pub fn register_iowq_aff(&self, cpu_set: &libc::cpu_set_t) -> io::Result<()> {
637        self.execute_register(
638            sys::IORING_REGISTER_IOWQ_AFF,
639            cpu_set as *const _ as *const libc::c_void,
640            mem::size_of::<libc::cpu_set_t>() as u32,
641        )
642        .map(drop)
643    }
644
645    /// Undoes a CPU mask previously set with register_iowq_aff
646    pub fn unregister_iowq_aff(&self) -> io::Result<()> {
647        self.execute_register(sys::IORING_UNREGISTER_IOWQ_AFF, ptr::null(), 0)
648            .map(drop)
649    }
650
651    /// Get and/or set the limit for number of io_uring worker threads per NUMA
652    /// node. `max[0]` holds the limit for bounded workers, which process I/O
653    /// operations expected to be bound in time, that is I/O on regular files or
654    /// block devices. While `max[1]` holds the limit for unbounded workers,
655    /// which carry out I/O operations that can never complete, for instance I/O
656    /// on sockets. Passing `0` does not change the current limit. Returns
657    /// previous limits on success.
658    pub fn register_iowq_max_workers(&self, max: &mut [u32; 2]) -> io::Result<()> {
659        self.execute_register(
660            sys::IORING_REGISTER_IOWQ_MAX_WORKERS,
661            max.as_mut_ptr().cast(),
662            max.len() as _,
663        )
664        .map(drop)
665    }
666
667    /// Register the io_uring instance's own file descriptor with the kernel, so that subsequent
668    /// [`enter`](Self::enter) calls (including [`submit`](Self::submit) and
669    /// [`submit_and_wait`](Self::submit_and_wait)) automatically pass
670    /// [`EnterFlags::REGISTERED_RING`] together with the registered index instead of the raw file
671    /// descriptor. This avoids the per-call file descriptor lookup overhead in the kernel.
672    ///
673    /// The registration is remembered by this [`Submitter`] and used transparently; call
674    /// [`unregister_ring_fd`](Self::unregister_ring_fd) on the same [`Submitter`] to undo it.
675    /// Calling this while a ring fd is already registered on this [`Submitter`] returns an `EEXIST`
676    /// error.
677    ///
678    /// On kernels with `IORING_FEAT_REG_REG_RING`, registration methods on this [`Submitter`] also
679    /// use the registered ring fd internally.
680    ///
681    /// Available since Linux 5.18.
682    pub fn register_ring_fd(&mut self) -> io::Result<()> {
683        if self.enter_ring_fd >= 0 {
684            return Err(io::Error::from_raw_os_error(libc::EEXIST));
685        }
686        let raw_fd = self.fd.as_raw_fd();
687        let mut up = sys::io_uring_rsrc_update {
688            offset: u32::MAX,
689            resv: 0,
690            data: raw_fd as _,
691        };
692        self.execute_register(
693            sys::IORING_REGISTER_RING_FDS,
694            (&mut up as *mut sys::io_uring_rsrc_update).cast(),
695            1,
696        )?;
697        self.enter_ring_fd = up.offset as i32;
698        Ok(())
699    }
700
701    /// Unregister a ring file descriptor previously registered with
702    /// [`register_ring_fd`](Self::register_ring_fd). Subsequent [`enter`](Self::enter) calls revert
703    /// to using the raw file descriptor, as do subsequent registration calls. Returns an `EINVAL`
704    /// error if no ring fd is registered.
705    ///
706    /// Available since Linux 5.18.
707    pub fn unregister_ring_fd(&mut self) -> io::Result<()> {
708        let offset = self.enter_ring_fd;
709        if offset < 0 {
710            return Err(io::Error::from_raw_os_error(libc::EINVAL));
711        }
712        let up = sys::io_uring_rsrc_update {
713            offset: offset as u32,
714            resv: 0,
715            data: 0,
716        };
717        self.execute_register(
718            sys::IORING_UNREGISTER_RING_FDS,
719            cast_ptr::<sys::io_uring_rsrc_update>(&up).cast(),
720            1,
721        )?;
722        self.enter_ring_fd = -1;
723        Ok(())
724    }
725
726    /// Register NAPI busy-poll settings on this ring.
727    ///
728    /// The kernel writes the previous settings back into `napi` before applying the new
729    /// ones; read them back with [`Napi::busy_poll_timeout`] and
730    /// [`Napi::prefer_busy_poll`].
731    ///
732    /// Available since Linux 6.9.
733    pub fn register_napi(&self, napi: &mut Napi) -> io::Result<()> {
734        self.execute_register(sys::IORING_REGISTER_NAPI, napi.as_mut_ptr().cast(), 1)
735            .map(drop)
736    }
737
738    /// Unregister NAPI busy-poll from this ring.
739    ///
740    /// The kernel writes the current settings back into `napi` before disabling them;
741    /// read them back with [`Napi::busy_poll_timeout`] and [`Napi::prefer_busy_poll`]. A
742    /// valid buffer is required, as the kernel rejects a null argument with `EINVAL`.
743    ///
744    /// Available since Linux 6.9.
745    pub fn unregister_napi(&self, napi: &mut Napi) -> io::Result<()> {
746        self.execute_register(sys::IORING_UNREGISTER_NAPI, napi.as_mut_ptr().cast(), 1)
747            .map(drop)
748    }
749
750    /// Add a NAPI id to this ring's statically tracked busy-poll set.
751    ///
752    /// The ring must already be registered with [`NapiTracking::Static`]; otherwise the
753    /// kernel returns an error. `napi_id` identifies a NIC receive-queue NAPI instance,
754    /// typically obtained from a socket via the `SO_INCOMING_NAPI_ID` socket option.
755    ///
756    /// [`NapiTracking::Static`]: crate::types::NapiTracking::Static
757    ///
758    /// Available since Linux 6.13.
759    pub fn register_napi_add_id(&self, napi_id: u32) -> io::Result<()> {
760        self.register_napi_static_op(sys::IO_URING_NAPI_STATIC_ADD_ID as _, napi_id)
761    }
762
763    /// Remove a NAPI id from this ring's statically tracked busy-poll set.
764    ///
765    /// The ring must already be registered with [`NapiTracking::Static`]; otherwise the
766    /// kernel returns an error. See [`register_napi_add_id`](Self::register_napi_add_id).
767    ///
768    /// [`NapiTracking::Static`]: crate::types::NapiTracking::Static
769    ///
770    /// Available since Linux 6.13.
771    pub fn register_napi_del_id(&self, napi_id: u32) -> io::Result<()> {
772        self.register_napi_static_op(sys::IO_URING_NAPI_STATIC_DEL_ID as _, napi_id)
773    }
774
775    fn register_napi_static_op(&self, opcode: u8, napi_id: u32) -> io::Result<()> {
776        // Both ops are issued through IORING_REGISTER_NAPI, distinguished by `opcode`,
777        // with the NAPI id carried in `op_param`. The kernel writes the current settings
778        // back into the struct, so pass a mutable pointer even though we discard them.
779        let mut arg = sys::io_uring_napi {
780            opcode,
781            op_param: napi_id,
782            ..Default::default()
783        };
784        self.execute_register(
785            sys::IORING_REGISTER_NAPI,
786            (&mut arg as *mut sys::io_uring_napi).cast(),
787            1,
788        )
789        .map(drop)
790    }
791
792    /// Register buffer ring for provided buffers.
793    ///
794    /// Details can be found in the io_uring_register_buf_ring.3 man page.
795    ///
796    /// If the register command is not supported, or the ring_entries value exceeds
797    /// 32768, the InvalidInput error is returned.
798    ///
799    /// Available since 5.19.
800    ///
801    /// # Safety
802    ///
803    /// Developers must ensure that the `ring_addr` and its length represented by `ring_entries`
804    /// are valid and will be valid until the bgid is unregistered or the ring destroyed,
805    /// otherwise undefined behaviour may occur.
806    #[deprecated(note = "please use `register_buf_ring_with_flags` instead")]
807    pub unsafe fn register_buf_ring(
808        &self,
809        ring_addr: u64,
810        ring_entries: u16,
811        bgid: u16,
812    ) -> io::Result<()> {
813        self.register_buf_ring_with_flags(ring_addr, ring_entries, bgid, 0)
814    }
815
816    /// Register buffer ring for provided buffers.
817    ///
818    /// Details can be found in the io_uring_register_buf_ring.3 man page.
819    ///
820    /// If the register command is not supported, or the ring_entries value exceeds
821    /// 32768, the InvalidInput error is returned.
822    ///
823    /// Available since 5.19.
824    ///
825    /// # Safety
826    ///
827    /// Developers must ensure that the `ring_addr` and its length represented by `ring_entries`
828    /// are valid and will be valid until the bgid is unregistered or the ring destroyed,
829    /// otherwise undefined behaviour may occur.
830    pub unsafe fn register_buf_ring_with_flags(
831        &self,
832        ring_addr: u64,
833        ring_entries: u16,
834        bgid: u16,
835        flags: u16,
836    ) -> io::Result<()> {
837        // The interface type for ring_entries is u32 but the same interface only allows a u16 for
838        // the tail to be specified, so to try and avoid further confusion, we limit the
839        // ring_entries to u16 here too. The value is actually limited to 2^15 (32768) but we can
840        // let the kernel enforce that.
841        let arg = sys::io_uring_buf_reg {
842            ring_addr,
843            ring_entries: ring_entries as _,
844            bgid,
845            flags,
846            ..Default::default()
847        };
848        self.execute_register(
849            sys::IORING_REGISTER_PBUF_RING,
850            cast_ptr::<sys::io_uring_buf_reg>(&arg).cast(),
851            1,
852        )
853        .map(drop)
854    }
855
856    /// Unregister a previously registered buffer ring.
857    ///
858    /// Available since 5.19.
859    pub fn unregister_buf_ring(&self, bgid: u16) -> io::Result<()> {
860        let arg = sys::io_uring_buf_reg {
861            ring_addr: 0,
862            ring_entries: 0,
863            bgid,
864            ..Default::default()
865        };
866        self.execute_register(
867            sys::IORING_UNREGISTER_PBUF_RING,
868            cast_ptr::<sys::io_uring_buf_reg>(&arg).cast(),
869            1,
870        )
871        .map(drop)
872    }
873
874    /// Performs a synchronous cancellation request, similar to [AsyncCancel](crate::opcode::AsyncCancel),
875    /// except that it completes synchronously.
876    ///
877    /// Cancellation can target a specific request, or all requests matching some criteria. The
878    /// [`CancelBuilder`] builder supports describing the match criteria for cancellation.
879    ///
880    /// An optional `timeout` can be provided to specify how long to wait for matched requests to be
881    /// canceled. If no timeout is provided, the default is to wait indefinitely.
882    ///
883    /// ### Errors
884    ///
885    /// If no requests are matched, returns:
886    ///
887    /// [io::ErrorKind::NotFound]: `No such file or directory (os error 2)`
888    ///
889    /// If a timeout is supplied, and the timeout elapses prior to all requests being canceled, returns:
890    ///
891    /// [io::ErrorKind::Uncategorized]: `Timer expired (os error 62)`
892    ///
893    /// ### Notes
894    ///
895    /// Only requests which have been submitted to the ring will be considered for cancellation. Requests
896    /// which have been written to the SQ, but not submitted, will not be canceled.
897    ///
898    /// Available since 6.0.
899    pub fn register_sync_cancel(
900        &self,
901        timeout: Option<Timespec>,
902        builder: CancelBuilder,
903    ) -> io::Result<()> {
904        let timespec = timeout.map(|ts| ts.0).unwrap_or(sys::__kernel_timespec {
905            tv_sec: -1,
906            tv_nsec: -1,
907        });
908        let user_data = builder.user_data.unwrap_or(0);
909        let flags = builder.flags.bits();
910        let fd = builder.to_fd();
911
912        let arg = sys::io_uring_sync_cancel_reg {
913            addr: user_data,
914            fd,
915            flags,
916            timeout: timespec,
917            ..Default::default()
918        };
919
920        self.execute_register(
921            sys::IORING_REGISTER_SYNC_CANCEL,
922            cast_ptr::<sys::io_uring_sync_cancel_reg>(&arg).cast(),
923            1,
924        )
925        .map(drop)
926    }
927
928    /// Register a netdev hw rx queue for zerocopy.
929    ///
930    /// Available since 6.15.
931    pub fn register_ifq(&self, reg: &sys::io_uring_zcrx_ifq_reg) -> io::Result<()> {
932        self.execute_register(
933            sys::IORING_REGISTER_ZCRX_IFQ,
934            cast_ptr::<sys::io_uring_zcrx_ifq_reg>(reg) as _,
935            1,
936        )
937        .map(drop)
938    }
939}