Skip to main content

store_io_posix/
lib.rs

1//! # store-io-posix
2//!
3//! The Linux backend of store-io. `PosixPlatform` implements
4//! `store_io_platform::Platform` over `O_DIRECT` file descriptors, on one of
5//! two tiers chosen at run time (`LinuxTier`, D-082): io_uring through
6//! store-io's own binding (`uring`: rings restricted to reads, writes, NOP
7//! and cancel, every read and write `RWF_NOWAIT`, what the kernel would block
8//! on finished by one ordinary system call; nothing ever runs on io-wq), or
9//! the synchronous tier (T3), where `PosixQueue` performs every data
10//! operation inside `submit` with one system call. A queue whose operations
11//! the caller waits for one at a time runs inline on either tier: no thread
12//! hop, no timer, no allocation.
13//!
14//! What this backend does:
15//!
16//! - opens data files `O_RDWR|O_DIRECT|O_CLOEXEC` (read-only:
17//!   `O_RDONLY|O_DIRECT|O_CLOEXEC`) relative to a directory descriptor, with
18//!   `openat2(RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS)` on kernels 5.6 and later
19//!   and `openat(O_NOFOLLOW)` before; symlinks are refused and so is anything
20//!   that is neither a regular file nor a block device;
21//! - creates files `O_CREAT|O_EXCL`, mode 0600, and directories mode 0700
22//!   followed by an `fsync` of the parent;
23//! - takes the ownership lock with `fcntl(F_OFD_SETLK)` on the very open file
24//!   description used for I/O (`flock` on kernels without OFD locks), never a
25//!   POSIX `F_SETLK`, which any `close()` of the inode would drop;
26//! - allocates with `fallocate` mode 0 only (never `KEEP_SIZE`); the engine
27//!   fills the unwritten extents with direct writes before a region is ready;
28//! - reports extents with `FS_IOC_FIEMAP` and page-cache residency with
29//!   `cachestat(2)`;
30//! - writes with `pwritev2(RWF_DSYNC)` when durability at completion is
31//!   requested, except on raw block devices, where the synchronous
32//!   `RWF_DSYNC` path issues both a FUA write and a cache flush; there a plain
33//!   write is followed by one `fdatasync`;
34//! - runs the operations of a parallel queue (`QueueConfig::parallel`) on a
35//!   bounded pool of worker threads, started on first use and parked untimed
36//!   while idle (D-051, D-096); every other queue stays inline;
37//! - gathers `Evidence` from statfs, statx, `/proc/self/mountinfo`, the ext4
38//!   effective options, inode flags, the sysfs block tree, NVMe Identify (when
39//!   the device node can be opened) and the DMI tables. Every sysfs, procfs
40//!   and device byte is parsed as hostile input. The probe never decides a
41//!   class and never writes a file.
42//!
43//! Errors are raw `errno` values; the engine classifies them. `EINTR` is
44//! retried, and the retry counted, only on opens, which provably precede any
45//! I/O. Data system calls on `O_DIRECT` files are not interrupted before I/O
46//! in practice, so their `EINTR` is returned as the raw outcome.
47//!
48//! Every module is gated on `target_os = "linux"`; on other operating systems
49//! the crate compiles to an empty shell so a workspace build succeeds
50//! everywhere. Every `unsafe` block lives in `sys.rs`, `posix.rs` and the io_uring
51//! binding (`uring/sys.rs`: its system calls and shared memory; `ring.rs`
52//! and `queue.rs`: pushing entries and the wake read's target), each preceded
53//! by a `SAFETY` comment citing the contract it relies on; tests aside.
54
55#![deny(warnings)]
56
57#[cfg(target_os = "linux")]
58mod alloc;
59#[cfg(target_os = "macos")]
60#[path = "mac/alloc.rs"]
61mod alloc;
62#[cfg(target_os = "linux")]
63mod dir;
64#[cfg(target_os = "macos")]
65#[path = "mac/dir.rs"]
66mod dir;
67#[cfg(target_os = "linux")]
68mod file;
69#[cfg(target_os = "macos")]
70#[path = "mac/file.rs"]
71mod file;
72/// Fuzz entry point for the probe's parsers (cargo-fuzz builds only).
73#[cfg(all(target_os = "linux", fuzzing))]
74#[doc(hidden)]
75pub mod fuzzing;
76#[cfg(target_os = "linux")]
77mod lock;
78#[cfg(target_os = "macos")]
79#[path = "mac/lock.rs"]
80mod lock;
81#[cfg(target_os = "macos")]
82#[path = "mac/platform.rs"]
83mod mac_platform;
84#[cfg(any(target_os = "linux", target_os = "macos"))]
85mod pool;
86#[cfg(any(target_os = "linux", target_os = "macos"))]
87mod posix;
88#[cfg(target_os = "linux")]
89mod probe;
90#[cfg(target_os = "macos")]
91#[path = "mac/probe.rs"]
92mod probe;
93#[cfg(any(target_os = "linux", target_os = "macos"))]
94mod queue;
95#[cfg(any(target_os = "linux", target_os = "macos"))]
96mod sync;
97#[cfg(target_os = "linux")]
98mod sys;
99#[cfg(target_os = "macos")]
100#[path = "mac/sys.rs"]
101mod sys;
102#[cfg(target_os = "linux")]
103mod uring;
104
105#[cfg(any(target_os = "linux", target_os = "macos"))]
106pub use dir::PosixDir;
107#[cfg(any(target_os = "linux", target_os = "macos"))]
108pub use file::PosixFile;
109#[cfg(any(target_os = "linux", target_os = "macos"))]
110pub use lock::PosixLock;
111#[cfg(target_os = "macos")]
112pub use mac_platform::PosixPlatform;
113#[cfg(target_os = "linux")]
114pub use queue::Registration;
115#[cfg(any(target_os = "linux", target_os = "macos"))]
116pub use queue::{PosixQueue, PosixWaker};
117
118#[cfg(target_os = "linux")]
119pub use store_io_platform::LinuxTier;
120
121/// Operations, in this process so far, that the io_uring tier finished with
122/// one blocking system call because the kernel would have blocked (or
123/// could not honour NOWAIT for the file) or transferred short. A measure
124/// of how often the asynchronous path was not taken; never a failure.
125#[cfg(target_os = "linux")]
126#[must_use]
127pub fn io_uring_fallbacks() -> u64 {
128    uring::queue::FALLBACKS.load(core::sync::atomic::Ordering::Relaxed)
129}
130
131#[cfg(target_os = "linux")]
132mod platform {
133    use core::sync::atomic::{AtomicU64, Ordering};
134    use std::path::Path;
135    use std::sync::{Arc, Mutex, OnceLock, PoisonError};
136
137    use store_io_platform::{
138        Evidence, FileMode, FileName, IoInterface, IoPath, OsError, Owner, Platform, QueueConfig,
139        RangeState, RawResult, ReleaseHow,
140    };
141
142    use super::LinuxTier;
143    use crate::uring::ring::{Flavour, Ring};
144
145    use crate::alloc;
146    use crate::dir::PosixDir;
147    use crate::file::PosixFile;
148    use crate::lock::PosixLock;
149    use crate::pool::{SubmitPool, default_threads};
150    use crate::probe;
151    use crate::queue::PosixQueue;
152    use crate::sys;
153
154    /// Upper bound on `EINTR` / `EAGAIN` retries of one open.
155    const OPEN_RETRY_LIMIT: u32 = 64;
156
157    /// The Linux synchronous platform.
158    ///
159    /// Cheap to create; probes once whether `openat2(2)` exists so every
160    /// confined open afterwards takes the right path without a syscall.
161    ///
162    /// # Examples
163    ///
164    /// ```no_run
165    /// use std::path::Path;
166    /// use store_io_platform::{FileName, Platform};
167    /// use store_io_posix::PosixPlatform;
168    ///
169    /// // Raw OS errors carry only the errno; the engine classifies them.
170    /// let p = PosixPlatform::new();
171    /// let dir = p.open_dir(Path::new("/var/lib/example"), true)?;
172    /// let name = FileName::new("store.sio")?;
173    /// let file = p.create_file(&dir, &name)?;
174    /// p.allocate(&file, 1 << 20)?;
175    /// let _lock = p.lock_exclusive(&file)?;
176    /// # Ok::<(), Box<dyn std::error::Error>>(())
177    /// ```
178    ///
179    /// Clones are handles to the same platform: they share the retry counters
180    /// and the submit pool.
181    #[derive(Debug, Clone)]
182    pub struct PosixPlatform {
183        openat2: bool,
184        eintr_retries: Arc<AtomicU64>,
185        eagain_retries: Arc<AtomicU64>,
186        /// The submit pool, started by the first parallel queue (D-096).
187        /// Parallel queues hold it too: its workers stop once the last
188        /// reference, platform or queue, is gone.
189        pool: Arc<Mutex<Option<Arc<SubmitPool>>>>,
190        /// `Sync` or `Uring` (never `Auto`: resolved at construction).
191        tier: LinuxTier,
192        /// Why `tier` was chosen.
193        reason: Arc<str>,
194        /// `Uring` came from `Auto`: a ring the kernel refuses (for want of
195        /// descriptors or memory, or a filter added since the probe) gives a
196        /// synchronous queue instead of an error.
197        auto: bool,
198        /// Queues that fell back so, and the last refusal's code (stored
199        /// before the count is raised).
200        fell_back: Arc<AtomicU64>,
201        /// Queues on rings.
202        ringed: Arc<AtomicU64>,
203        /// Rings the kernel refused with an error to the caller (every
204        /// refusal under an explicit `Uring`; under `Auto`, one a
205        /// synchronous queue cannot get around): a submitter is not created,
206        /// a parallel lane's operations (or an open's header reads) run on
207        /// the store's lone-operation lanes. `last_refusal` holds the latest
208        /// code of either kind.
209        refused: Arc<AtomicU64>,
210        last_refusal: Arc<core::sync::atomic::AtomicI32>,
211        /// Rings asked to register (D-078) and what they got.
212        registration: Arc<RegistrationCounts>,
213    }
214
215    /// Rings asked to register the file, and how many got it.
216    #[derive(Debug, Default)]
217    struct RegistrationCounts {
218        asked: AtomicU64,
219        files: AtomicU64,
220        /// The latest refusal's code (0: none).
221        refusal: core::sync::atomic::AtomicI32,
222    }
223
224    /// Whether this kernel accepts store-io's rings of both flavours; `Err`
225    /// holds the refusal and a description. An answer that cannot change
226    /// (accepted, or refused as unsupported or forbidden) is kept for the
227    /// process; a refusal for want of resources is asked again next time.
228    fn uring_available() -> Result<(), (OsError, String)> {
229        static PROBE: OnceLock<Result<(), (OsError, String)>> = OnceLock::new();
230        if let Some(known) = PROBE.get() {
231            return known.clone();
232        }
233        let answer = probe_rings();
234        let lasting = match &answer {
235            Ok(()) => true,
236            Err((e, _)) => [
237                libc::ENOSYS,
238                libc::EINVAL,
239                libc::EPERM,
240                libc::EACCES,
241                libc::EOPNOTSUPP,
242            ]
243            .contains(&e.code),
244        };
245        if lasting {
246            let _first = PROBE.set(answer.clone());
247        }
248        answer
249    }
250
251    /// A ring refusal a synchronous queue gets around: no descriptors or
252    /// memory, or a filter (seccomp, `io_uring_disabled`) added since the
253    /// probe. Any other code (a bad argument, say) is a fault to report.
254    fn passes_to_sync(e: OsError) -> bool {
255        [
256            libc::EMFILE,
257            libc::ENFILE,
258            libc::ENOMEM,
259            libc::EAGAIN,
260            libc::EPERM,
261            libc::EACCES,
262            libc::ENOSYS,
263        ]
264        .contains(&e.code)
265    }
266
267    fn probe_rings() -> Result<(), (OsError, String)> {
268        for (flavour, name) in [(Flavour::Shared, "shared"), (Flavour::Owned, "owned")] {
269            if let Err(e) = Ring::new(flavour, 1) {
270                let knob = std::fs::read_to_string("/proc/sys/kernel/io_uring_disabled")
271                    .ok()
272                    .map(|v| v.trim().to_owned())
273                    .filter(|v| v != "0")
274                    .map_or(String::new(), |v| {
275                        format!("; kernel.io_uring_disabled = {v}")
276                    });
277                return Err((e, format!("io_uring refused a {name} ring ({e}){knob}")));
278            }
279        }
280        Ok(())
281    }
282
283    impl Default for PosixPlatform {
284        fn default() -> Self {
285            Self::new()
286        }
287    }
288
289    impl PosixPlatform {
290        /// Creates the platform, probing `openat2(2)` support once.
291        #[must_use]
292        pub fn new() -> Self {
293            Self {
294                openat2: sys::openat2_supported(),
295                eintr_retries: Arc::new(AtomicU64::new(0)),
296                eagain_retries: Arc::new(AtomicU64::new(0)),
297                pool: Arc::new(Mutex::new(None)),
298                tier: LinuxTier::Sync,
299                reason: Arc::from("synchronous by default (PosixPlatform::new)"),
300                auto: false,
301                fell_back: Arc::new(AtomicU64::new(0)),
302                refused: Arc::new(AtomicU64::new(0)),
303                registration: Arc::default(),
304                ringed: Arc::new(AtomicU64::new(0)),
305                last_refusal: Arc::new(core::sync::atomic::AtomicI32::new(0)),
306            }
307        }
308
309        /// A platform whose queues use `tier`: [`LinuxTier::Auto`] takes
310        /// io_uring where the kernel accepts store-io's rings of both
311        /// flavours (probed once per process), else the synchronous tier.
312        /// The choice and its reason are [`Platform::io_path`]. Under `Auto`,
313        /// a queue whose ring the kernel refuses later (for want of
314        /// descriptors or memory, or a filter installed since the probe) is
315        /// a synchronous queue instead, counted in the reason at the time it
316        /// is read (and the interface is synchronous if every queue fell
317        /// back); any other refusal, and every refusal under `Uring`, is an
318        /// error.
319        ///
320        /// # Errors
321        ///
322        /// For [`LinuxTier::Uring`] only: the kernel's refusal to set up a
323        /// ring (io_uring disabled, too old, or without a feature store-io
324        /// requires: `ENOSYS`).
325        pub fn with_tier(tier: LinuxTier) -> RawResult<Self> {
326            let auto = !matches!(tier, LinuxTier::Sync | LinuxTier::Uring);
327            let (tier, reason) = match tier {
328                LinuxTier::Sync => (LinuxTier::Sync, "chosen".to_owned()),
329                LinuxTier::Uring => match uring_available() {
330                    Ok(()) => (LinuxTier::Uring, "chosen".to_owned()),
331                    Err((e, _)) => return Err(e),
332                },
333                _ => match uring_available() {
334                    Ok(()) => (
335                        LinuxTier::Uring,
336                        "auto: the kernel accepts both ring flavours".to_owned(),
337                    ),
338                    Err((_, why)) => (LinuxTier::Sync, format!("auto: {why}")),
339                },
340            };
341            Ok(Self {
342                tier,
343                reason: Arc::from(reason),
344                auto: auto && tier == LinuxTier::Uring,
345                ..Self::new()
346            })
347        }
348
349        /// The tier this platform's queues use (`Sync` or `Uring`).
350        #[must_use]
351        pub fn tier(&self) -> LinuxTier {
352            self.tier
353        }
354
355        /// Worker threads in the submit pool, or `None` before the first
356        /// parallel queue starts it. Its size is fixed by that first queue
357        /// (`QueueConfig::pool_threads`, at most 256; else one per CPU up to
358        /// 32).
359        #[must_use]
360        pub fn pool_threads(&self) -> Option<usize> {
361            self.pool
362                .lock()
363                .unwrap_or_else(PoisonError::into_inner)
364                .as_ref()
365                .map(|p| p.threads())
366        }
367
368        /// The submit pool, started with `threads` workers (0 = the default)
369        /// if it is not running yet. A running pool is never resized.
370        fn submit_pool(&self, threads: u32) -> RawResult<Arc<SubmitPool>> {
371            let mut slot = self.pool.lock().unwrap_or_else(PoisonError::into_inner);
372            if let Some(p) = slot.as_ref() {
373                return Ok(Arc::clone(p));
374            }
375            let n = match usize::try_from(threads) {
376                Ok(0) | Err(_) => default_threads(),
377                Ok(n) => n,
378            };
379            let p = Arc::new(SubmitPool::start(n)?);
380            *slot = Some(Arc::clone(&p));
381            Ok(p)
382        }
383
384        /// Whether confined opens use `openat2(RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS)`
385        /// (kernel 5.6 and later) rather than `openat(O_NOFOLLOW)`.
386        #[must_use]
387        pub fn openat2_supported(&self) -> bool {
388            self.openat2
389        }
390
391        /// Opens retried after `EINTR` since creation (each retry counted).
392        #[must_use]
393        pub fn eintr_retries(&self) -> u64 {
394            self.eintr_retries.load(Ordering::Relaxed)
395        }
396
397        /// `openat2` calls retried after a `RESOLVE_BENEATH` race (`EAGAIN`)
398        /// since creation.
399        #[must_use]
400        pub fn eagain_retries(&self) -> u64 {
401            self.eagain_retries.load(Ordering::Relaxed)
402        }
403
404        /// Runs an open, retrying `EINTR` (and `EAGAIN` when `openat2` is in
405        /// use) a bounded number of times. Both precede any I/O.
406        pub(crate) fn retry_open<T>(&self, mut open: impl FnMut() -> RawResult<T>) -> RawResult<T> {
407            let mut attempts = 0;
408            loop {
409                match open() {
410                    Err(e) if e.code == libc::EINTR && attempts < OPEN_RETRY_LIMIT => {
411                        attempts += 1;
412                        let _count = self.eintr_retries.fetch_add(1, Ordering::Relaxed);
413                    }
414                    Err(e)
415                        if e.code == libc::EAGAIN
416                            && self.openat2
417                            && attempts < OPEN_RETRY_LIMIT =>
418                    {
419                        attempts += 1;
420                        let _count = self.eagain_retries.fetch_add(1, Ordering::Relaxed);
421                    }
422                    r => return r,
423                }
424            }
425        }
426    }
427
428    impl Platform for PosixPlatform {
429        type File = PosixFile;
430        type Dir = PosixDir;
431        type Lock = PosixLock;
432        type Queue = PosixQueue;
433
434        fn open_dir(&self, path: &Path, create: bool) -> RawResult<PosixDir> {
435            PosixDir::open(self, path, create)
436        }
437
438        fn create_file(&self, dir: &PosixDir, name: &FileName) -> RawResult<PosixFile> {
439            PosixFile::create(self, dir, name)
440        }
441
442        fn open_file(
443            &self,
444            dir: &PosixDir,
445            name: &FileName,
446            mode: FileMode,
447        ) -> RawResult<PosixFile> {
448            PosixFile::open(self, dir, name, mode)
449        }
450
451        fn rename_replace(
452            &self,
453            dir: &PosixDir,
454            from: &FileName,
455            to: &FileName,
456            _file: &PosixFile,
457        ) -> RawResult<()> {
458            dir.rename(from, to)
459        }
460
461        fn rename_noreplace(
462            &self,
463            dir: &PosixDir,
464            from: &FileName,
465            to: &FileName,
466            _file: &PosixFile,
467        ) -> RawResult<()> {
468            dir.rename_noreplace(from, to)
469        }
470
471        fn unlink(&self, dir: &PosixDir, name: &FileName) -> RawResult<()> {
472            dir.unlink(name)
473        }
474
475        fn sync_dir(&self, dir: &PosixDir) -> RawResult<()> {
476            dir.sync()
477        }
478
479        fn lock_exclusive(&self, file: &PosixFile) -> RawResult<PosixLock> {
480            PosixLock::acquire(file, false)
481        }
482
483        fn lock_shared(&self, file: &PosixFile) -> RawResult<PosixLock> {
484            PosixLock::acquire(file, true)
485        }
486
487        fn flush_errors_per_handle(&self) -> bool {
488            // errseq_t, per open file description, since Linux 4.13; since
489            // 4.16 a description opened after an unseen error still sees it
490            // (the extra slots' handles are opened after the store's own).
491            sys::kernel_release()
492                .as_deref()
493                .and_then(crate::probe::virt::kernel_version)
494                .is_some_and(|v| v >= (4, 16, 0))
495        }
496
497        fn reopen(&self, file: &PosixFile) -> Option<PosixFile> {
498            file.read_write_twin().ok()
499        }
500
501        fn monotonic_ms(&self) -> u64 {
502            sys::monotonic_coarse_ms()
503        }
504
505        fn io_path(&self) -> IoPath {
506            let fell_back = self.fell_back.load(Ordering::Acquire);
507            let refused = self.refused.load(Ordering::Acquire);
508            let ringed = self.ringed.load(Ordering::Relaxed);
509            let mut reason = self.reason.to_string();
510            let code = self.last_refusal.load(Ordering::Relaxed);
511            if fell_back > 0 {
512                reason.push_str(&format!(
513                    "; {fell_back} queue(s) synchronous: the kernel refused a ring ({})",
514                    sys::errno(code)
515                ));
516            }
517            let r = &self.registration;
518            let asked = r.asked.load(Ordering::Relaxed);
519            if asked > 0 {
520                reason.push_str(&format!(
521                    "; registration (D-078) asked on {asked} ring(s): a direct descriptor on {}",
522                    r.files.load(Ordering::Relaxed)
523                ));
524                let refusal = r.refusal.load(Ordering::Relaxed);
525                if refusal != 0 {
526                    reason.push_str(&format!(" (refused: {})", sys::errno(refusal)));
527                }
528            }
529            if refused > 0 {
530                reason.push_str(&format!(
531                    "; {refused} ring(s) refused by the kernel ({}): a submitter needing one was not created, or a parallel lane's (or an open's header reads') operations ran one at a time",
532                    sys::errno(code)
533                ));
534            }
535            IoPath {
536                // Every queue so far fell back: the store is synchronous.
537                interface: if self.tier == LinuxTier::Uring && (ringed > 0 || fell_back == 0) {
538                    IoInterface::IoUring
539                } else {
540                    IoInterface::Synchronous
541                },
542                reason,
543            }
544        }
545
546        fn allocate(&self, file: &PosixFile, len: u64) -> RawResult<()> {
547            alloc::allocate(file, len)
548        }
549
550        fn allocate_range(&self, file: &PosixFile, offset: u64, len: u64) -> RawResult<()> {
551            alloc::allocate_range(file, offset, len)
552        }
553
554        fn set_len(&self, file: &PosixFile, len: u64) -> RawResult<()> {
555            alloc::set_len(file, len)
556        }
557
558        fn size(&self, file: &PosixFile) -> RawResult<u64> {
559            alloc::size(file)
560        }
561
562        fn release_range(
563            &self,
564            file: &PosixFile,
565            offset: u64,
566            len: u64,
567            how: ReleaseHow,
568        ) -> RawResult<()> {
569            alloc::release(file, offset, len, how)
570        }
571
572        fn range_state(&self, file: &PosixFile, offset: u64, len: u64) -> RawResult<RangeState> {
573            alloc::range_state(file, offset, len)
574        }
575
576        fn flush_all(&self, file: &PosixFile) -> RawResult<()> {
577            sys::fsync(file.raw())
578        }
579
580        fn probe(&self, dir: &PosixDir, file: &PosixFile) -> RawResult<Evidence> {
581            probe::probe(dir, file)
582        }
583
584        fn queue(&self, cfg: QueueConfig) -> RawResult<PosixQueue> {
585            self.make_queue(cfg, None)
586        }
587
588        fn queue_registered(&self, cfg: QueueConfig, file: &PosixFile) -> RawResult<PosixQueue> {
589            // Only a thread-owned ring registers (R3.3): a read-only twin of
590            // the file, opened only once a ring is made (`file::read_only_twin`).
591            let reg = (cfg.owner == Owner::Thread).then(|| crate::uring::queue::Registered {
592                file: file.share_fd(),
593            });
594            self.make_queue(cfg, reg)
595        }
596    }
597
598    impl PosixPlatform {
599        fn make_queue(
600            &self,
601            cfg: QueueConfig,
602            reg: Option<crate::uring::queue::Registered>,
603        ) -> RawResult<PosixQueue> {
604            // Rings for queues that keep several operations in flight (a
605            // `Submitter`, a parallel lane). A queue whose caller runs one
606            // operation at a time and waits for it (`parallel: false`, a
607            // blocking call's lane) does it inline with one system call on
608            // the io_uring tier too: a ring round trip for a lone operation
609            // costs more than the call (D-104), and buys nothing.
610            let ring = cfg.parallel || cfg.owner == Owner::Thread;
611            if self.tier == LinuxTier::Uring && ring {
612                let owned = cfg.owner == Owner::Thread;
613                let asked = reg.is_some();
614                match PosixQueue::uring(cfg, owned, reg) {
615                    Ok(q) => {
616                        let _n = self.ringed.fetch_add(1, Ordering::Relaxed);
617                        if asked {
618                            self.count_registration(&q);
619                        }
620                        return Ok(q);
621                    }
622                    // `Auto` chose rings; this one was refused: a synchronous
623                    // queue, which needs nothing from the kernel, serves
624                    // (below), and is counted.
625                    Err(e) if self.auto && passes_to_sync(e) => {
626                        self.last_refusal.store(e.code, Ordering::Relaxed);
627                        let _n = self.fell_back.fetch_add(1, Ordering::Release);
628                    }
629                    Err(e) => {
630                        self.last_refusal.store(e.code, Ordering::Relaxed);
631                        let _n = self.refused.fetch_add(1, Ordering::Release);
632                        return Err(e);
633                    }
634                }
635            }
636            if cfg.parallel {
637                Ok(PosixQueue::pooled(cfg, self.submit_pool(cfg.pool_threads)?))
638            } else {
639                Ok(PosixQueue::new(cfg))
640            }
641        }
642
643        /// Counts what a ring asked to register got (D-078), for `io_path`.
644        fn count_registration(&self, q: &PosixQueue) {
645            let Some(got) = q.registered() else {
646                return;
647            };
648            let r = &self.registration;
649            let _a = r.asked.fetch_add(1, Ordering::Relaxed);
650            if got.file {
651                let _f = r.files.fetch_add(1, Ordering::Relaxed);
652            }
653            if let Some(e) = got.refused {
654                r.refusal.store(e.code, Ordering::Relaxed);
655            }
656        }
657    }
658}
659
660#[cfg(target_os = "linux")]
661pub use platform::PosixPlatform;