io_uring/submit.rs
1use std::os::unix::io::{AsRawFd, RawFd};
2use std::sync::atomic;
3use std::{io, mem, ptr};
4
5use crate::register::{execute, Probe, RegisterRing};
6use crate::sys;
7use crate::types::{CancelBuilder, CloneBuffersFlags, Napi, Timespec};
8use crate::util::{cast_ptr, OwnedFd};
9use crate::Parameters;
10use bitflags::bitflags;
11
12use crate::register::Restriction;
13
14use crate::types;
15
16bitflags!(
17 /// See man page for complete description:
18 /// https://man7.org/linux/man-pages/man2/io_uring_enter.2.html
19 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
20 pub struct EnterFlags: u32 {
21 /// Wait for at least `min_complete` events to complete.
22 const GETEVENTS = sys::IORING_ENTER_GETEVENTS;
23
24 /// If the kernel thread is sleeping, wake it up.
25 const SQ_WAKEUP = sys::IORING_ENTER_SQ_WAKEUP;
26
27 /// Wait for at least one submission queue entry to be available.
28 const SQ_WAIT = sys::IORING_ENTER_SQ_WAIT;
29
30 /// Use the extended argument structure.
31 const EXT_ARG = sys::IORING_ENTER_EXT_ARG;
32
33 /// Submit using registered submission queue ring.
34 const REGISTERED_RING = sys::IORING_ENTER_REGISTERED_RING;
35
36 /// Timeout argument interpreted as absolute time.
37 const ABS_TIMER = sys::IORING_ENTER_ABS_TIMER;
38
39 /// Arg is offset into an area of wait regions previously registered.
40 const EXT_ARG_REG = sys::IORING_ENTER_EXT_ARG_REG;
41
42 /// Don't mark waiting task as being in iowait in certain cases.
43 const NO_IOWAIT = sys::IORING_ENTER_NO_IOWAIT;
44 }
45);
46
47/// Interface for submitting submission queue events in an io_uring instance to the kernel for
48/// executing and registering files or buffers with the instance.
49///
50/// io_uring supports both directly performing I/O on buffers and file descriptors and registering
51/// them beforehand. Registering is slow, but it makes performing the actual I/O much faster.
52pub struct Submitter<'a> {
53 fd: &'a OwnedFd,
54 params: &'a Parameters,
55 enter_ring_fd: i32,
56
57 sq_head: *const atomic::AtomicU32,
58 sq_tail: *const atomic::AtomicU32,
59 sq_flags: *const atomic::AtomicU32,
60}
61
62impl<'a> Submitter<'a> {
63 #[inline]
64 pub(crate) const fn new(
65 fd: &'a OwnedFd,
66 params: &'a Parameters,
67 sq_head: *const atomic::AtomicU32,
68 sq_tail: *const atomic::AtomicU32,
69 sq_flags: *const atomic::AtomicU32,
70 ) -> Submitter<'a> {
71 Submitter {
72 fd,
73 params,
74 enter_ring_fd: -1,
75 sq_head,
76 sq_tail,
77 sq_flags,
78 }
79 }
80
81 #[inline]
82 fn sq_len(&self) -> usize {
83 unsafe {
84 let head = (*self.sq_head).load(atomic::Ordering::Acquire);
85 let tail = (*self.sq_tail).load(atomic::Ordering::Acquire);
86
87 tail.wrapping_sub(head) as usize
88 }
89 }
90
91 /// Whether the kernel thread has gone to sleep because it waited for too long without
92 /// submission queue entries.
93 #[inline]
94 fn sq_need_wakeup(&self) -> bool {
95 unsafe {
96 (*self.sq_flags).load(atomic::Ordering::Relaxed) & sys::IORING_SQ_NEED_WAKEUP != 0
97 }
98 }
99
100 /// Load flags published by the kernel in the submission queue ring.
101 #[inline]
102 fn load_sq_flags(&self) -> u32 {
103 unsafe { (*self.sq_flags).load(atomic::Ordering::Acquire) }
104 }
105
106 #[inline]
107 fn execute_register(
108 &self,
109 opcode: libc::c_uint,
110 arg: *const libc::c_void,
111 len: libc::c_uint,
112 ) -> io::Result<i32> {
113 execute(self.register_ring(), opcode, arg, len)
114 }
115
116 #[inline]
117 fn register_ring(&self) -> RegisterRing {
118 let enter_ring_fd = self.enter_ring_fd;
119 if enter_ring_fd >= 0 && self.params.is_feature_reg_reg_ring() {
120 RegisterRing::RegisteredIndex(enter_ring_fd)
121 } else {
122 RegisterRing::RawFd(self.fd.as_raw_fd())
123 }
124 }
125
126 /// Initiate and/or complete asynchronous I/O. This is a low-level wrapper around
127 /// `io_uring_enter` - see `man io_uring_enter` (or [its online
128 /// version](https://manpages.debian.org/unstable/liburing-dev/io_uring_enter.2.en.html) for
129 /// more details.
130 ///
131 /// You will probably want to use a more high-level API such as
132 /// [`submit`](Self::submit) or [`submit_and_wait`](Self::submit_and_wait).
133 ///
134 /// # Safety
135 ///
136 /// This provides a raw interface so developer must ensure that parameters are correct.
137 pub unsafe fn enter<T: Sized>(
138 &self,
139 to_submit: u32,
140 min_complete: u32,
141 flag: u32,
142 arg: Option<&T>,
143 ) -> io::Result<usize> {
144 let arg = arg
145 .map(|arg| cast_ptr(arg).cast())
146 .unwrap_or_else(ptr::null);
147 let size = mem::size_of::<T>();
148
149 // If a ring fd has been registered with [`register_ring_fd`](Self::register_ring_fd),
150 // `enter_ring_fd` holds its index (otherwise `-1`); pass it together with
151 // `IORING_ENTER_REGISTERED_RING` instead of the raw file descriptor to avoid the per-call
152 // fd lookup in the kernel.
153 let enter_ring_fd = self.enter_ring_fd;
154 let (fd, flag) = if enter_ring_fd >= 0 {
155 (enter_ring_fd, flag | sys::IORING_ENTER_REGISTERED_RING)
156 } else {
157 (self.fd.as_raw_fd(), flag)
158 };
159
160 sys::io_uring_enter(fd, to_submit, min_complete, flag, arg, size).map(|res| res as _)
161 }
162
163 /// Submit all queued submission queue events to the kernel.
164 #[inline]
165 pub fn submit(&self) -> io::Result<usize> {
166 self.submit_and_wait(0)
167 }
168
169 /// Submit all queued submission queue events to the kernel and wait for at least `want`
170 /// completion events to complete.
171 pub fn submit_and_wait(&self, want: usize) -> io::Result<usize> {
172 let len = self.sq_len();
173 let mut flags = EnterFlags::empty();
174
175 // This logic suffers from the fact the sq_cq_overflow and sq_need_wakeup
176 // each cause an atomic load of the same variable, self.sq_flags.
177 // In the hottest paths, when a server is running with sqpoll,
178 // this is going to be hit twice, when once would be sufficient.
179 // However, consider that the `SeqCst` barrier required for interpreting
180 // the IORING_ENTER_SQ_WAKEUP bit is required in all paths where sqpoll
181 // is setup when consolidating the reads.
182
183 let sq_flags = self.load_sq_flags();
184 let sq_cq_overflow = sq_flags & sys::IORING_SQ_CQ_OVERFLOW != 0;
185 let sq_taskrun = sq_flags & sys::IORING_SQ_TASKRUN != 0;
186
187 // When IORING_FEAT_NODROP is enabled and CQ overflows, the kernel buffers
188 // completion events internally but doesn't automatically flush them when
189 // CQ space becomes available. We must explicitly call io_uring_enter()
190 // to flush these buffered events, even with SQPOLL enabled.
191 //
192 // Without this, completions remain stuck in kernel's internal buffer
193 // after draining CQ, causing missing completion notifications.
194 let need_syscall = (sq_cq_overflow && self.params.is_feature_nodrop()) || sq_taskrun;
195
196 // Deferred task work is only run by an enter carrying GETEVENTS.
197 if want > 0 || self.params.is_setup_iopoll() || sq_cq_overflow || sq_taskrun {
198 flags.insert(EnterFlags::GETEVENTS);
199 }
200
201 if self.params.is_setup_sqpoll() {
202 // See discussion in [`SubmissionQueue::need_wakeup`].
203 atomic::fence(atomic::Ordering::SeqCst);
204 if self.sq_need_wakeup() {
205 flags.insert(EnterFlags::SQ_WAKEUP);
206 } else if want == 0 && !need_syscall {
207 // The kernel thread is polling and hasn't fallen asleep, so we don't need to tell
208 // it to process events or wake it up
209
210 // However, if the CQ ring is overflown, we need to tell the kernel to process events
211 // by calling io_uring_enter with the IORING_ENTER_GETEVENTS flag.
212 return Ok(len);
213 }
214 }
215
216 unsafe { self.enter::<libc::sigset_t>(len as _, want as _, flags.bits(), None) }
217 }
218
219 /// Submit all queued submission queue events to the kernel and wait for at least `want`
220 /// completion events to complete with additional options
221 ///
222 /// You can specify a set of signals to mask and a timeout for operation, see
223 /// [`SubmitArgs`](types::SubmitArgs) for more details
224 pub fn submit_with_args(
225 &self,
226 want: usize,
227 args: &types::SubmitArgs<'_, '_>,
228 ) -> io::Result<usize> {
229 self.submit_with_args_and_flags(want, args, EnterFlags::empty())
230 }
231
232 /// Like [`submit_with_args`](Self::submit_with_args), with extra `flags` for the wait.
233 ///
234 /// Only these flags are accepted; any other fails with `EINVAL`, without entering the kernel:
235 ///
236 /// - [`EnterFlags::ABS_TIMER`]: the timeout in `args` is an absolute time on the ring's clock
237 /// (`CLOCK_MONOTONIC` unless registered otherwise), not a duration. Available since kernel
238 /// 6.12.
239 /// - [`EnterFlags::NO_IOWAIT`]: the wait is not accounted as iowait. See
240 /// [`Parameters::is_feature_no_iowait`](crate::Parameters::is_feature_no_iowait).
241 ///
242 /// A kernel that doesn't know a flag fails with `EINVAL`.
243 pub fn submit_with_args_and_flags(
244 &self,
245 want: usize,
246 args: &types::SubmitArgs<'_, '_>,
247 flags: EnterFlags,
248 ) -> io::Result<usize> {
249 if !(EnterFlags::ABS_TIMER | EnterFlags::NO_IOWAIT).contains(flags) {
250 return Err(io::Error::from_raw_os_error(libc::EINVAL));
251 }
252
253 let len = self.sq_len();
254 let mut flags = flags | EnterFlags::EXT_ARG;
255
256 let sq_flags = self.load_sq_flags();
257 let sq_cq_overflow = sq_flags & sys::IORING_SQ_CQ_OVERFLOW != 0;
258 let sq_taskrun = sq_flags & sys::IORING_SQ_TASKRUN != 0;
259 let need_syscall = (sq_cq_overflow && self.params.is_feature_nodrop()) || sq_taskrun;
260
261 // Deferred task work is only run by an enter carrying GETEVENTS.
262 if want > 0 || self.params.is_setup_iopoll() || sq_cq_overflow || sq_taskrun {
263 flags.insert(EnterFlags::GETEVENTS);
264 }
265
266 if self.params.is_setup_sqpoll() {
267 // See discussion in [`SubmissionQueue::need_wakeup`].
268 atomic::fence(atomic::Ordering::SeqCst);
269 if self.sq_need_wakeup() {
270 flags.insert(EnterFlags::SQ_WAKEUP);
271 } else if want == 0 && !need_syscall {
272 // The kernel thread is polling and hasn't fallen asleep, so we don't need to tell
273 // it to process events or wake it up
274 return Ok(len);
275 }
276 }
277
278 unsafe { self.enter(len as _, want as _, flags.bits(), Some(args)) }
279 }
280
281 /// Wait for the submission queue to have free entries.
282 pub fn squeue_wait(&self) -> io::Result<usize> {
283 unsafe { self.enter::<libc::sigset_t>(0, 0, EnterFlags::SQ_WAIT.bits(), None) }
284 }
285
286 /// Register in-memory fixed buffers for I/O with the kernel. You can use these buffers with the
287 /// [`ReadFixed`](crate::opcode::ReadFixed) and [`WriteFixed`](crate::opcode::WriteFixed)
288 /// operations.
289 ///
290 /// # Safety
291 ///
292 /// Developers must ensure that the `iov_base` and `iov_len` values are valid and will
293 /// be valid until buffers are unregistered or the ring destroyed, otherwise undefined
294 /// behaviour may occur.
295 pub unsafe fn register_buffers(&self, bufs: &[libc::iovec]) -> io::Result<()> {
296 self.execute_register(
297 sys::IORING_REGISTER_BUFFERS,
298 bufs.as_ptr().cast(),
299 bufs.len() as _,
300 )
301 .map(drop)
302 }
303
304 /// Update a range of fixed buffers starting at `offset`.
305 ///
306 /// This is required to use buffers registered using
307 /// [`register_buffers_sparse`](Self::register_buffers_sparse),
308 /// although it can be also be used with [`register_buffers`](Self::register_buffers).
309 ///
310 /// See [`register_buffers2`](Self::register_buffers2)
311 /// for more information about resource tagging.
312 ///
313 /// Available since Linux 5.13.
314 ///
315 /// # Safety
316 ///
317 /// Developers must ensure that the `iov_base` and `iov_len` values are valid and will
318 /// be valid until buffers are unregistered or the ring destroyed, otherwise undefined
319 /// behaviour may occur.
320 pub unsafe fn register_buffers_update(
321 &self,
322 offset: u32,
323 bufs: &[libc::iovec],
324 tags: Option<&[u64]>,
325 ) -> io::Result<()> {
326 let nr = tags
327 .as_ref()
328 .map_or(bufs.len(), |tags| bufs.len().min(tags.len()));
329
330 let rr = sys::io_uring_rsrc_update2 {
331 nr: nr as _,
332 data: bufs.as_ptr() as _,
333 tags: tags.map(|tags| tags.as_ptr() as _).unwrap_or(0),
334 offset,
335 ..Default::default()
336 };
337
338 self.execute_register(
339 sys::IORING_REGISTER_BUFFERS_UPDATE,
340 cast_ptr::<sys::io_uring_rsrc_update2>(&rr).cast(),
341 std::mem::size_of::<sys::io_uring_rsrc_update2>() as _,
342 )
343 .map(drop)
344 }
345
346 /// Variant of [`register_buffers`](Self::register_buffers)
347 /// with resource tagging.
348 ///
349 /// `tags` should be the same length as `bufs` and contain the
350 /// tag value corresponding to the buffer at the same index.
351 ///
352 /// If a tag is zero, then tagging for this particular resource
353 /// (a buffer in this case) is disabled. Otherwise, after the
354 /// resource had been unregistered and it's not used anymore,
355 /// a CQE will be posted with `user_data` set to the specified
356 /// tag and all other fields zeroed.
357 ///
358 /// Available since Linux 5.13.
359 ///
360 /// # Safety
361 ///
362 /// Developers must ensure that the `iov_base` and `iov_len` values are valid and will
363 /// be valid until buffers are unregistered or the ring destroyed, otherwise undefined
364 /// behaviour may occur.
365 pub unsafe fn register_buffers2(&self, bufs: &[libc::iovec], tags: &[u64]) -> io::Result<()> {
366 let rr = sys::io_uring_rsrc_register {
367 nr: bufs.len().min(tags.len()) as _,
368 data: bufs.as_ptr() as _,
369 tags: tags.as_ptr() as _,
370 ..Default::default()
371 };
372 self.execute_register(
373 sys::IORING_REGISTER_BUFFERS2,
374 cast_ptr::<sys::io_uring_rsrc_register>(&rr).cast(),
375 std::mem::size_of::<sys::io_uring_rsrc_register>() as _,
376 )
377 .map(drop)
378 }
379
380 /// Registers an empty table of nr fixed buffers buffers.
381 ///
382 /// These must be updated before use, using eg.
383 /// [`register_buffers_update`](Self::register_buffers_update).
384 ///
385 /// See [`register_buffers`](Self::register_buffers)
386 /// for more information about fixed buffers.
387 ///
388 /// Available since Linux 5.13.
389 pub fn register_buffers_sparse(&self, nr: u32) -> io::Result<()> {
390 let rr = sys::io_uring_rsrc_register {
391 nr,
392 flags: sys::IORING_RSRC_REGISTER_SPARSE,
393 ..Default::default()
394 };
395 self.execute_register(
396 sys::IORING_REGISTER_BUFFERS2,
397 cast_ptr::<sys::io_uring_rsrc_register>(&rr).cast(),
398 std::mem::size_of::<sys::io_uring_rsrc_register>() as _,
399 )
400 .map(drop)
401 }
402
403 /// Clone the entire registered buffer table from another ring into this one.
404 ///
405 /// `src_fd` is the raw file descriptor of the source `io_uring`. The source's
406 /// buffers are shared with this ring rather than copied, so a single physical
407 /// registration can back many rings without re-pinning the pages in the kernel.
408 ///
409 /// This ring's buffer table must be empty. To clone into a non-empty table or
410 /// to copy a sub-range, use
411 /// [`register_buffers_clone_offset`](Self::register_buffers_clone_offset).
412 ///
413 /// Available since Linux 6.12.
414 pub fn register_buffers_clone(&self, src_fd: RawFd) -> io::Result<()> {
415 self.register_buffers_clone_offset(src_fd, 0, 0, 0, CloneBuffersFlags::empty())
416 }
417
418 /// Clone a range of the registered buffer table from another ring into this one.
419 ///
420 /// `src_fd` is the raw file descriptor of the source `io_uring`. `nr` buffers
421 /// starting at `src_off` in the source table are installed starting at `dst_off`
422 /// in this ring's table. A `nr` of `0` clones the source's entire table.
423 ///
424 /// See [`CloneBuffersFlags`] for replacing an existing destination range or
425 /// treating `src_fd` as a registered ring descriptor.
426 ///
427 /// Available since Linux 6.12.
428 pub fn register_buffers_clone_offset(
429 &self,
430 src_fd: RawFd,
431 src_off: u32,
432 dst_off: u32,
433 nr: u32,
434 flags: CloneBuffersFlags,
435 ) -> io::Result<()> {
436 let arg = sys::io_uring_clone_buffers {
437 src_fd: src_fd as _,
438 flags: flags.bits(),
439 src_off,
440 dst_off,
441 nr,
442 ..Default::default()
443 };
444 execute(
445 RegisterRing::RawFd(self.fd.as_raw_fd()),
446 sys::IORING_REGISTER_CLONE_BUFFERS,
447 cast_ptr::<sys::io_uring_clone_buffers>(&arg).cast(),
448 // This opcode takes a single struct; the kernel requires nr_args == 1.
449 1,
450 )
451 .map(drop)
452 }
453
454 /// Registers an empty file table of nr_files number of file descriptors. The sparse variant is
455 /// available in kernels 5.19 and later.
456 ///
457 /// Registering a file table is a prerequisite for using any request that
458 /// uses direct descriptors.
459 pub fn register_files_sparse(&self, nr: u32) -> io::Result<()> {
460 let rr = sys::io_uring_rsrc_register {
461 nr,
462 flags: sys::IORING_RSRC_REGISTER_SPARSE,
463 resv2: 0,
464 data: 0,
465 tags: 0,
466 };
467 self.execute_register(
468 sys::IORING_REGISTER_FILES2,
469 cast_ptr::<sys::io_uring_rsrc_register>(&rr).cast(),
470 mem::size_of::<sys::io_uring_rsrc_register>() as _,
471 )
472 .map(drop)
473 }
474
475 /// Register files for I/O. You can use the registered files with
476 /// [`Fixed`](crate::types::Fixed).
477 ///
478 /// Each fd may be -1, in which case it is considered "sparse", and can be filled in later with
479 /// [`register_files_update`](Self::register_files_update).
480 ///
481 /// Note that this will wait for the ring to idle; it will only return once all active requests
482 /// are complete. Use [`register_files_update`](Self::register_files_update) to avoid this.
483 pub fn register_files(&self, fds: &[RawFd]) -> io::Result<()> {
484 self.execute_register(
485 sys::IORING_REGISTER_FILES,
486 fds.as_ptr().cast(),
487 fds.len() as _,
488 )
489 .map(drop)
490 }
491
492 /// This operation replaces existing files in the registered file set with new ones,
493 /// either turning a sparse entry (one where fd is equal to -1) into a real one, removing an existing entry (new one is set to -1),
494 /// or replacing an existing entry with a new existing entry. The `offset` parameter specifies
495 /// the offset into the list of registered files at which to start updating files.
496 ///
497 /// You can also perform this asynchronously with the
498 /// [`FilesUpdate`](crate::opcode::FilesUpdate) opcode.
499 pub fn register_files_update(&self, offset: u32, fds: &[RawFd]) -> io::Result<usize> {
500 let fu = sys::io_uring_files_update {
501 offset,
502 resv: 0,
503 fds: fds.as_ptr() as _,
504 };
505 let ret = self.execute_register(
506 sys::IORING_REGISTER_FILES_UPDATE,
507 cast_ptr::<sys::io_uring_files_update>(&fu).cast(),
508 fds.len() as _,
509 )?;
510 Ok(ret as _)
511 }
512
513 /// Register an eventfd created by [`eventfd`](libc::eventfd) with the io_uring instance.
514 pub fn register_eventfd(&self, eventfd: RawFd) -> io::Result<()> {
515 self.execute_register(
516 sys::IORING_REGISTER_EVENTFD,
517 cast_ptr::<RawFd>(&eventfd).cast(),
518 1,
519 )
520 .map(drop)
521 }
522
523 /// This works just like [`register_eventfd`](Self::register_eventfd), except notifications are
524 /// only posted for events that complete in an async manner, so requests that complete
525 /// immediately will not cause a notification.
526 pub fn register_eventfd_async(&self, eventfd: RawFd) -> io::Result<()> {
527 self.execute_register(
528 sys::IORING_REGISTER_EVENTFD_ASYNC,
529 cast_ptr::<RawFd>(&eventfd).cast(),
530 1,
531 )
532 .map(drop)
533 }
534
535 /// Fill in the given [`Probe`] with information about the opcodes supported by io_uring on the
536 /// running kernel.
537 ///
538 /// # Examples
539 ///
540 // This is marked no_run as it is only available from Linux 5.6+, however the latest Ubuntu (on
541 // which CI runs) only has Linux 5.4.
542 /// ```no_run
543 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
544 /// let io_uring = io_uring::IoUring::new(1)?;
545 /// let mut probe = io_uring::Probe::new();
546 /// io_uring.submitter().register_probe(&mut probe)?;
547 ///
548 /// if probe.is_supported(io_uring::opcode::Read::CODE) {
549 /// println!("Reading is supported!");
550 /// }
551 /// # Ok(())
552 /// # }
553 /// ```
554 pub fn register_probe(&self, probe: &mut Probe) -> io::Result<()> {
555 self.execute_register(
556 sys::IORING_REGISTER_PROBE,
557 probe.as_mut_ptr() as *const _,
558 Probe::COUNT as _,
559 )
560 .map(drop)
561 }
562
563 /// Register credentials of the running application with io_uring, and get an id associated with
564 /// these credentials. This ID can then be [passed](crate::squeue::Entry::personality) into
565 /// submission queue entries to issue the request with this process' credentials.
566 ///
567 /// By default, if [`Parameters::is_feature_cur_personality`] is set then requests will use the
568 /// credentials of the task that called [`Submitter::enter`], otherwise they will use the
569 /// credentials of the task that originally registered the io_uring.
570 ///
571 /// [`Parameters::is_feature_cur_personality`]: crate::Parameters::is_feature_cur_personality
572 pub fn register_personality(&self) -> io::Result<u16> {
573 let id = self.execute_register(sys::IORING_REGISTER_PERSONALITY, ptr::null(), 0)?;
574 Ok(id as u16)
575 }
576
577 /// Unregister all previously registered buffers.
578 ///
579 /// You do not need to explicitly call this before dropping the [`IoUring`](crate::IoUring), as
580 /// it will be cleaned up by the kernel automatically.
581 ///
582 /// Available since Linux 5.1.
583 pub fn unregister_buffers(&self) -> io::Result<()> {
584 self.execute_register(sys::IORING_UNREGISTER_BUFFERS, ptr::null(), 0)
585 .map(drop)
586 }
587
588 /// Unregister all previously registered files.
589 ///
590 /// You do not need to explicitly call this before dropping the [`IoUring`](crate::IoUring), as
591 /// it will be cleaned up by the kernel automatically.
592 pub fn unregister_files(&self) -> io::Result<()> {
593 self.execute_register(sys::IORING_UNREGISTER_FILES, ptr::null(), 0)
594 .map(drop)
595 }
596
597 /// Unregister an eventfd file descriptor to stop notifications.
598 pub fn unregister_eventfd(&self) -> io::Result<()> {
599 self.execute_register(sys::IORING_UNREGISTER_EVENTFD, ptr::null(), 0)
600 .map(drop)
601 }
602
603 /// Unregister a previously registered personality.
604 pub fn unregister_personality(&self, personality: u16) -> io::Result<()> {
605 self.execute_register(
606 sys::IORING_UNREGISTER_PERSONALITY,
607 ptr::null(),
608 personality as _,
609 )
610 .map(drop)
611 }
612
613 /// Permanently install a feature allowlist. Once this has been called, attempting to perform
614 /// an operation not on the allowlist will fail with `-EACCES`.
615 ///
616 /// This can only be called once, to prevent untrusted code from removing restrictions.
617 pub fn register_restrictions(&self, res: &mut [Restriction]) -> io::Result<()> {
618 self.execute_register(
619 sys::IORING_REGISTER_RESTRICTIONS,
620 res.as_mut_ptr().cast(),
621 res.len() as _,
622 )
623 .map(drop)
624 }
625
626 /// Enable the rings of the io_uring instance if they have been disabled with
627 /// [`setup_r_disabled`](crate::Builder::setup_r_disabled).
628 pub fn register_enable_rings(&self) -> io::Result<()> {
629 self.execute_register(sys::IORING_REGISTER_ENABLE_RINGS, ptr::null(), 0)
630 .map(drop)
631 }
632
633 /// Tell io_uring on what CPUs the async workers can run. By default, async workers
634 /// created by io_uring will inherit the CPU mask of its parent. This is usually
635 /// all the CPUs in the system, unless the parent is being run with a limited set.
636 pub fn register_iowq_aff(&self, cpu_set: &libc::cpu_set_t) -> io::Result<()> {
637 self.execute_register(
638 sys::IORING_REGISTER_IOWQ_AFF,
639 cpu_set as *const _ as *const libc::c_void,
640 mem::size_of::<libc::cpu_set_t>() as u32,
641 )
642 .map(drop)
643 }
644
645 /// Undoes a CPU mask previously set with register_iowq_aff
646 pub fn unregister_iowq_aff(&self) -> io::Result<()> {
647 self.execute_register(sys::IORING_UNREGISTER_IOWQ_AFF, ptr::null(), 0)
648 .map(drop)
649 }
650
651 /// Get and/or set the limit for number of io_uring worker threads per NUMA
652 /// node. `max[0]` holds the limit for bounded workers, which process I/O
653 /// operations expected to be bound in time, that is I/O on regular files or
654 /// block devices. While `max[1]` holds the limit for unbounded workers,
655 /// which carry out I/O operations that can never complete, for instance I/O
656 /// on sockets. Passing `0` does not change the current limit. Returns
657 /// previous limits on success.
658 pub fn register_iowq_max_workers(&self, max: &mut [u32; 2]) -> io::Result<()> {
659 self.execute_register(
660 sys::IORING_REGISTER_IOWQ_MAX_WORKERS,
661 max.as_mut_ptr().cast(),
662 max.len() as _,
663 )
664 .map(drop)
665 }
666
667 /// Register the io_uring instance's own file descriptor with the kernel, so that subsequent
668 /// [`enter`](Self::enter) calls (including [`submit`](Self::submit) and
669 /// [`submit_and_wait`](Self::submit_and_wait)) automatically pass
670 /// [`EnterFlags::REGISTERED_RING`] together with the registered index instead of the raw file
671 /// descriptor. This avoids the per-call file descriptor lookup overhead in the kernel.
672 ///
673 /// The registration is remembered by this [`Submitter`] and used transparently; call
674 /// [`unregister_ring_fd`](Self::unregister_ring_fd) on the same [`Submitter`] to undo it.
675 /// Calling this while a ring fd is already registered on this [`Submitter`] returns an `EEXIST`
676 /// error.
677 ///
678 /// On kernels with `IORING_FEAT_REG_REG_RING`, registration methods on this [`Submitter`] also
679 /// use the registered ring fd internally.
680 ///
681 /// Available since Linux 5.18.
682 pub fn register_ring_fd(&mut self) -> io::Result<()> {
683 if self.enter_ring_fd >= 0 {
684 return Err(io::Error::from_raw_os_error(libc::EEXIST));
685 }
686 let raw_fd = self.fd.as_raw_fd();
687 let mut up = sys::io_uring_rsrc_update {
688 offset: u32::MAX,
689 resv: 0,
690 data: raw_fd as _,
691 };
692 self.execute_register(
693 sys::IORING_REGISTER_RING_FDS,
694 (&mut up as *mut sys::io_uring_rsrc_update).cast(),
695 1,
696 )?;
697 self.enter_ring_fd = up.offset as i32;
698 Ok(())
699 }
700
701 /// Unregister a ring file descriptor previously registered with
702 /// [`register_ring_fd`](Self::register_ring_fd). Subsequent [`enter`](Self::enter) calls revert
703 /// to using the raw file descriptor, as do subsequent registration calls. Returns an `EINVAL`
704 /// error if no ring fd is registered.
705 ///
706 /// Available since Linux 5.18.
707 pub fn unregister_ring_fd(&mut self) -> io::Result<()> {
708 let offset = self.enter_ring_fd;
709 if offset < 0 {
710 return Err(io::Error::from_raw_os_error(libc::EINVAL));
711 }
712 let up = sys::io_uring_rsrc_update {
713 offset: offset as u32,
714 resv: 0,
715 data: 0,
716 };
717 self.execute_register(
718 sys::IORING_UNREGISTER_RING_FDS,
719 cast_ptr::<sys::io_uring_rsrc_update>(&up).cast(),
720 1,
721 )?;
722 self.enter_ring_fd = -1;
723 Ok(())
724 }
725
726 /// Register NAPI busy-poll settings on this ring.
727 ///
728 /// The kernel writes the previous settings back into `napi` before applying the new
729 /// ones; read them back with [`Napi::busy_poll_timeout`] and
730 /// [`Napi::prefer_busy_poll`].
731 ///
732 /// Available since Linux 6.9.
733 pub fn register_napi(&self, napi: &mut Napi) -> io::Result<()> {
734 self.execute_register(sys::IORING_REGISTER_NAPI, napi.as_mut_ptr().cast(), 1)
735 .map(drop)
736 }
737
738 /// Unregister NAPI busy-poll from this ring.
739 ///
740 /// The kernel writes the current settings back into `napi` before disabling them;
741 /// read them back with [`Napi::busy_poll_timeout`] and [`Napi::prefer_busy_poll`]. A
742 /// valid buffer is required, as the kernel rejects a null argument with `EINVAL`.
743 ///
744 /// Available since Linux 6.9.
745 pub fn unregister_napi(&self, napi: &mut Napi) -> io::Result<()> {
746 self.execute_register(sys::IORING_UNREGISTER_NAPI, napi.as_mut_ptr().cast(), 1)
747 .map(drop)
748 }
749
750 /// Add a NAPI id to this ring's statically tracked busy-poll set.
751 ///
752 /// The ring must already be registered with [`NapiTracking::Static`]; otherwise the
753 /// kernel returns an error. `napi_id` identifies a NIC receive-queue NAPI instance,
754 /// typically obtained from a socket via the `SO_INCOMING_NAPI_ID` socket option.
755 ///
756 /// [`NapiTracking::Static`]: crate::types::NapiTracking::Static
757 ///
758 /// Available since Linux 6.13.
759 pub fn register_napi_add_id(&self, napi_id: u32) -> io::Result<()> {
760 self.register_napi_static_op(sys::IO_URING_NAPI_STATIC_ADD_ID as _, napi_id)
761 }
762
763 /// Remove a NAPI id from this ring's statically tracked busy-poll set.
764 ///
765 /// The ring must already be registered with [`NapiTracking::Static`]; otherwise the
766 /// kernel returns an error. See [`register_napi_add_id`](Self::register_napi_add_id).
767 ///
768 /// [`NapiTracking::Static`]: crate::types::NapiTracking::Static
769 ///
770 /// Available since Linux 6.13.
771 pub fn register_napi_del_id(&self, napi_id: u32) -> io::Result<()> {
772 self.register_napi_static_op(sys::IO_URING_NAPI_STATIC_DEL_ID as _, napi_id)
773 }
774
775 fn register_napi_static_op(&self, opcode: u8, napi_id: u32) -> io::Result<()> {
776 // Both ops are issued through IORING_REGISTER_NAPI, distinguished by `opcode`,
777 // with the NAPI id carried in `op_param`. The kernel writes the current settings
778 // back into the struct, so pass a mutable pointer even though we discard them.
779 let mut arg = sys::io_uring_napi {
780 opcode,
781 op_param: napi_id,
782 ..Default::default()
783 };
784 self.execute_register(
785 sys::IORING_REGISTER_NAPI,
786 (&mut arg as *mut sys::io_uring_napi).cast(),
787 1,
788 )
789 .map(drop)
790 }
791
792 /// Register buffer ring for provided buffers.
793 ///
794 /// Details can be found in the io_uring_register_buf_ring.3 man page.
795 ///
796 /// If the register command is not supported, or the ring_entries value exceeds
797 /// 32768, the InvalidInput error is returned.
798 ///
799 /// Available since 5.19.
800 ///
801 /// # Safety
802 ///
803 /// Developers must ensure that the `ring_addr` and its length represented by `ring_entries`
804 /// are valid and will be valid until the bgid is unregistered or the ring destroyed,
805 /// otherwise undefined behaviour may occur.
806 #[deprecated(note = "please use `register_buf_ring_with_flags` instead")]
807 pub unsafe fn register_buf_ring(
808 &self,
809 ring_addr: u64,
810 ring_entries: u16,
811 bgid: u16,
812 ) -> io::Result<()> {
813 self.register_buf_ring_with_flags(ring_addr, ring_entries, bgid, 0)
814 }
815
816 /// Register buffer ring for provided buffers.
817 ///
818 /// Details can be found in the io_uring_register_buf_ring.3 man page.
819 ///
820 /// If the register command is not supported, or the ring_entries value exceeds
821 /// 32768, the InvalidInput error is returned.
822 ///
823 /// Available since 5.19.
824 ///
825 /// # Safety
826 ///
827 /// Developers must ensure that the `ring_addr` and its length represented by `ring_entries`
828 /// are valid and will be valid until the bgid is unregistered or the ring destroyed,
829 /// otherwise undefined behaviour may occur.
830 pub unsafe fn register_buf_ring_with_flags(
831 &self,
832 ring_addr: u64,
833 ring_entries: u16,
834 bgid: u16,
835 flags: u16,
836 ) -> io::Result<()> {
837 // The interface type for ring_entries is u32 but the same interface only allows a u16 for
838 // the tail to be specified, so to try and avoid further confusion, we limit the
839 // ring_entries to u16 here too. The value is actually limited to 2^15 (32768) but we can
840 // let the kernel enforce that.
841 let arg = sys::io_uring_buf_reg {
842 ring_addr,
843 ring_entries: ring_entries as _,
844 bgid,
845 flags,
846 ..Default::default()
847 };
848 self.execute_register(
849 sys::IORING_REGISTER_PBUF_RING,
850 cast_ptr::<sys::io_uring_buf_reg>(&arg).cast(),
851 1,
852 )
853 .map(drop)
854 }
855
856 /// Unregister a previously registered buffer ring.
857 ///
858 /// Available since 5.19.
859 pub fn unregister_buf_ring(&self, bgid: u16) -> io::Result<()> {
860 let arg = sys::io_uring_buf_reg {
861 ring_addr: 0,
862 ring_entries: 0,
863 bgid,
864 ..Default::default()
865 };
866 self.execute_register(
867 sys::IORING_UNREGISTER_PBUF_RING,
868 cast_ptr::<sys::io_uring_buf_reg>(&arg).cast(),
869 1,
870 )
871 .map(drop)
872 }
873
874 /// Performs a synchronous cancellation request, similar to [AsyncCancel](crate::opcode::AsyncCancel),
875 /// except that it completes synchronously.
876 ///
877 /// Cancellation can target a specific request, or all requests matching some criteria. The
878 /// [`CancelBuilder`] builder supports describing the match criteria for cancellation.
879 ///
880 /// An optional `timeout` can be provided to specify how long to wait for matched requests to be
881 /// canceled. If no timeout is provided, the default is to wait indefinitely.
882 ///
883 /// ### Errors
884 ///
885 /// If no requests are matched, returns:
886 ///
887 /// [io::ErrorKind::NotFound]: `No such file or directory (os error 2)`
888 ///
889 /// If a timeout is supplied, and the timeout elapses prior to all requests being canceled, returns:
890 ///
891 /// [io::ErrorKind::Uncategorized]: `Timer expired (os error 62)`
892 ///
893 /// ### Notes
894 ///
895 /// Only requests which have been submitted to the ring will be considered for cancellation. Requests
896 /// which have been written to the SQ, but not submitted, will not be canceled.
897 ///
898 /// Available since 6.0.
899 pub fn register_sync_cancel(
900 &self,
901 timeout: Option<Timespec>,
902 builder: CancelBuilder,
903 ) -> io::Result<()> {
904 let timespec = timeout.map(|ts| ts.0).unwrap_or(sys::__kernel_timespec {
905 tv_sec: -1,
906 tv_nsec: -1,
907 });
908 let user_data = builder.user_data.unwrap_or(0);
909 let flags = builder.flags.bits();
910 let fd = builder.to_fd();
911
912 let arg = sys::io_uring_sync_cancel_reg {
913 addr: user_data,
914 fd,
915 flags,
916 timeout: timespec,
917 ..Default::default()
918 };
919
920 self.execute_register(
921 sys::IORING_REGISTER_SYNC_CANCEL,
922 cast_ptr::<sys::io_uring_sync_cancel_reg>(&arg).cast(),
923 1,
924 )
925 .map(drop)
926 }
927
928 /// Register a netdev hw rx queue for zerocopy.
929 ///
930 /// Available since 6.15.
931 pub fn register_ifq(&self, reg: &sys::io_uring_zcrx_ifq_reg) -> io::Result<()> {
932 self.execute_register(
933 sys::IORING_REGISTER_ZCRX_IFQ,
934 cast_ptr::<sys::io_uring_zcrx_ifq_reg>(reg) as _,
935 1,
936 )
937 .map(drop)
938 }
939}