store_io_posix/lib.rs
1//! # store-io-posix
2//!
3//! The Linux backend of store-io. `PosixPlatform` implements
4//! `store_io_platform::Platform` over `O_DIRECT` file descriptors, on one of
5//! two tiers chosen at run time (`LinuxTier`, D-082): io_uring through
6//! store-io's own binding (`uring`: rings restricted to reads, writes, NOP
7//! and cancel, every read and write `RWF_NOWAIT`, what the kernel would block
8//! on finished by one ordinary system call; nothing ever runs on io-wq), or
9//! the synchronous tier (T3), where `PosixQueue` performs every data
10//! operation inside `submit` with one system call. A queue whose operations
11//! the caller waits for one at a time runs inline on either tier: no thread
12//! hop, no timer, no allocation.
13//!
14//! What this backend does:
15//!
16//! - opens data files `O_RDWR|O_DIRECT|O_CLOEXEC` (read-only:
17//! `O_RDONLY|O_DIRECT|O_CLOEXEC`) relative to a directory descriptor, with
18//! `openat2(RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS)` on kernels 5.6 and later
19//! and `openat(O_NOFOLLOW)` before; symlinks are refused and so is anything
20//! that is neither a regular file nor a block device;
21//! - creates files `O_CREAT|O_EXCL`, mode 0600, and directories mode 0700
22//! followed by an `fsync` of the parent;
23//! - takes the ownership lock with `fcntl(F_OFD_SETLK)` on the very open file
24//! description used for I/O (`flock` on kernels without OFD locks), never a
25//! POSIX `F_SETLK`, which any `close()` of the inode would drop;
26//! - allocates with `fallocate` mode 0 only (never `KEEP_SIZE`); the engine
27//! fills the unwritten extents with direct writes before a region is ready;
28//! - reports extents with `FS_IOC_FIEMAP` and page-cache residency with
29//! `cachestat(2)`;
30//! - writes with `pwritev2(RWF_DSYNC)` when durability at completion is
31//! requested, except on raw block devices, where the synchronous
32//! `RWF_DSYNC` path issues both a FUA write and a cache flush; there a plain
33//! write is followed by one `fdatasync`;
34//! - runs the operations of a parallel queue (`QueueConfig::parallel`) on a
35//! bounded pool of worker threads, started on first use and parked untimed
36//! while idle (D-051, D-096); every other queue stays inline;
37//! - gathers `Evidence` from statfs, statx, `/proc/self/mountinfo`, the ext4
38//! effective options, inode flags, the sysfs block tree, NVMe Identify (when
39//! the device node can be opened) and the DMI tables. Every sysfs, procfs
40//! and device byte is parsed as hostile input. The probe never decides a
41//! class and never writes a file.
42//!
43//! Errors are raw `errno` values; the engine classifies them. `EINTR` is
44//! retried, and the retry counted, only on opens, which provably precede any
45//! I/O. Data system calls on `O_DIRECT` files are not interrupted before I/O
46//! in practice, so their `EINTR` is returned as the raw outcome.
47//!
48//! Every module is gated on `target_os = "linux"`; on other operating systems
49//! the crate compiles to an empty shell so a workspace build succeeds
50//! everywhere. Every `unsafe` block lives in `sys.rs`, `posix.rs` and the io_uring
51//! binding (`uring/sys.rs`: its system calls and shared memory; `ring.rs`
52//! and `queue.rs`: pushing entries and the wake read's target), each preceded
53//! by a `SAFETY` comment citing the contract it relies on; tests aside.
54
55#![deny(warnings)]
56
57#[cfg(target_os = "linux")]
58mod alloc;
59#[cfg(target_os = "macos")]
60#[path = "mac/alloc.rs"]
61mod alloc;
62#[cfg(target_os = "linux")]
63mod dir;
64#[cfg(target_os = "macos")]
65#[path = "mac/dir.rs"]
66mod dir;
67#[cfg(target_os = "linux")]
68mod file;
69#[cfg(target_os = "macos")]
70#[path = "mac/file.rs"]
71mod file;
72/// Fuzz entry point for the probe's parsers (cargo-fuzz builds only).
73#[cfg(all(target_os = "linux", fuzzing))]
74#[doc(hidden)]
75pub mod fuzzing;
76#[cfg(target_os = "linux")]
77mod lock;
78#[cfg(target_os = "macos")]
79#[path = "mac/lock.rs"]
80mod lock;
81#[cfg(target_os = "macos")]
82#[path = "mac/platform.rs"]
83mod mac_platform;
84#[cfg(any(target_os = "linux", target_os = "macos"))]
85mod pool;
86#[cfg(any(target_os = "linux", target_os = "macos"))]
87mod posix;
88#[cfg(target_os = "linux")]
89mod probe;
90#[cfg(target_os = "macos")]
91#[path = "mac/probe.rs"]
92mod probe;
93#[cfg(any(target_os = "linux", target_os = "macos"))]
94mod queue;
95#[cfg(any(target_os = "linux", target_os = "macos"))]
96mod sync;
97#[cfg(target_os = "linux")]
98mod sys;
99#[cfg(target_os = "macos")]
100#[path = "mac/sys.rs"]
101mod sys;
102#[cfg(target_os = "linux")]
103mod uring;
104
105#[cfg(any(target_os = "linux", target_os = "macos"))]
106pub use dir::PosixDir;
107#[cfg(any(target_os = "linux", target_os = "macos"))]
108pub use file::PosixFile;
109#[cfg(any(target_os = "linux", target_os = "macos"))]
110pub use lock::PosixLock;
111#[cfg(target_os = "macos")]
112pub use mac_platform::PosixPlatform;
113#[cfg(target_os = "linux")]
114pub use queue::Registration;
115#[cfg(any(target_os = "linux", target_os = "macos"))]
116pub use queue::{PosixQueue, PosixWaker};
117
118#[cfg(target_os = "linux")]
119pub use store_io_platform::LinuxTier;
120
121/// Operations, in this process so far, that the io_uring tier finished with
122/// one blocking system call because the kernel would have blocked (or
123/// could not honour NOWAIT for the file) or transferred short. A measure
124/// of how often the asynchronous path was not taken; never a failure.
125#[cfg(target_os = "linux")]
126#[must_use]
127pub fn io_uring_fallbacks() -> u64 {
128 uring::queue::FALLBACKS.load(core::sync::atomic::Ordering::Relaxed)
129}
130
131#[cfg(target_os = "linux")]
132mod platform {
133 use core::sync::atomic::{AtomicU64, Ordering};
134 use std::path::Path;
135 use std::sync::{Arc, Mutex, OnceLock, PoisonError};
136
137 use store_io_platform::{
138 Evidence, FileMode, FileName, IoInterface, IoPath, OsError, Owner, Platform, QueueConfig,
139 RangeState, RawResult, ReleaseHow,
140 };
141
142 use super::LinuxTier;
143 use crate::uring::ring::{Flavour, Ring};
144
145 use crate::alloc;
146 use crate::dir::PosixDir;
147 use crate::file::PosixFile;
148 use crate::lock::PosixLock;
149 use crate::pool::{SubmitPool, default_threads};
150 use crate::probe;
151 use crate::queue::PosixQueue;
152 use crate::sys;
153
154 /// Upper bound on `EINTR` / `EAGAIN` retries of one open.
155 const OPEN_RETRY_LIMIT: u32 = 64;
156
157 /// The Linux synchronous platform.
158 ///
159 /// Cheap to create; probes once whether `openat2(2)` exists so every
160 /// confined open afterwards takes the right path without a syscall.
161 ///
162 /// # Examples
163 ///
164 /// ```no_run
165 /// use std::path::Path;
166 /// use store_io_platform::{FileName, Platform};
167 /// use store_io_posix::PosixPlatform;
168 ///
169 /// // Raw OS errors carry only the errno; the engine classifies them.
170 /// let p = PosixPlatform::new();
171 /// let dir = p.open_dir(Path::new("/var/lib/example"), true)?;
172 /// let name = FileName::new("store.sio")?;
173 /// let file = p.create_file(&dir, &name)?;
174 /// p.allocate(&file, 1 << 20)?;
175 /// let _lock = p.lock_exclusive(&file)?;
176 /// # Ok::<(), Box<dyn std::error::Error>>(())
177 /// ```
178 ///
179 /// Clones are handles to the same platform: they share the retry counters
180 /// and the submit pool.
181 #[derive(Debug, Clone)]
182 pub struct PosixPlatform {
183 openat2: bool,
184 eintr_retries: Arc<AtomicU64>,
185 eagain_retries: Arc<AtomicU64>,
186 /// The submit pool, started by the first parallel queue (D-096).
187 /// Parallel queues hold it too: its workers stop once the last
188 /// reference, platform or queue, is gone.
189 pool: Arc<Mutex<Option<Arc<SubmitPool>>>>,
190 /// `Sync` or `Uring` (never `Auto`: resolved at construction).
191 tier: LinuxTier,
192 /// Why `tier` was chosen.
193 reason: Arc<str>,
194 /// `Uring` came from `Auto`: a ring the kernel refuses (for want of
195 /// descriptors or memory, or a filter added since the probe) gives a
196 /// synchronous queue instead of an error.
197 auto: bool,
198 /// Queues that fell back so, and the last refusal's code (stored
199 /// before the count is raised).
200 fell_back: Arc<AtomicU64>,
201 /// Queues on rings.
202 ringed: Arc<AtomicU64>,
203 /// Rings the kernel refused with an error to the caller (every
204 /// refusal under an explicit `Uring`; under `Auto`, one a
205 /// synchronous queue cannot get around): a submitter is not created,
206 /// a parallel lane's operations (or an open's header reads) run on
207 /// the store's lone-operation lanes. `last_refusal` holds the latest
208 /// code of either kind.
209 refused: Arc<AtomicU64>,
210 last_refusal: Arc<core::sync::atomic::AtomicI32>,
211 /// Rings asked to register (D-078) and what they got.
212 registration: Arc<RegistrationCounts>,
213 }
214
215 /// Rings asked to register the file, and how many got it.
216 #[derive(Debug, Default)]
217 struct RegistrationCounts {
218 asked: AtomicU64,
219 files: AtomicU64,
220 /// The latest refusal's code (0: none).
221 refusal: core::sync::atomic::AtomicI32,
222 }
223
224 /// Whether this kernel accepts store-io's rings of both flavours; `Err`
225 /// holds the refusal and a description. An answer that cannot change
226 /// (accepted, or refused as unsupported or forbidden) is kept for the
227 /// process; a refusal for want of resources is asked again next time.
228 fn uring_available() -> Result<(), (OsError, String)> {
229 static PROBE: OnceLock<Result<(), (OsError, String)>> = OnceLock::new();
230 if let Some(known) = PROBE.get() {
231 return known.clone();
232 }
233 let answer = probe_rings();
234 let lasting = match &answer {
235 Ok(()) => true,
236 Err((e, _)) => [
237 libc::ENOSYS,
238 libc::EINVAL,
239 libc::EPERM,
240 libc::EACCES,
241 libc::EOPNOTSUPP,
242 ]
243 .contains(&e.code),
244 };
245 if lasting {
246 let _first = PROBE.set(answer.clone());
247 }
248 answer
249 }
250
251 /// A ring refusal a synchronous queue gets around: no descriptors or
252 /// memory, or a filter (seccomp, `io_uring_disabled`) added since the
253 /// probe. Any other code (a bad argument, say) is a fault to report.
254 fn passes_to_sync(e: OsError) -> bool {
255 [
256 libc::EMFILE,
257 libc::ENFILE,
258 libc::ENOMEM,
259 libc::EAGAIN,
260 libc::EPERM,
261 libc::EACCES,
262 libc::ENOSYS,
263 ]
264 .contains(&e.code)
265 }
266
267 fn probe_rings() -> Result<(), (OsError, String)> {
268 for (flavour, name) in [(Flavour::Shared, "shared"), (Flavour::Owned, "owned")] {
269 if let Err(e) = Ring::new(flavour, 1) {
270 let knob = std::fs::read_to_string("/proc/sys/kernel/io_uring_disabled")
271 .ok()
272 .map(|v| v.trim().to_owned())
273 .filter(|v| v != "0")
274 .map_or(String::new(), |v| {
275 format!("; kernel.io_uring_disabled = {v}")
276 });
277 return Err((e, format!("io_uring refused a {name} ring ({e}){knob}")));
278 }
279 }
280 Ok(())
281 }
282
283 impl Default for PosixPlatform {
284 fn default() -> Self {
285 Self::new()
286 }
287 }
288
289 impl PosixPlatform {
290 /// Creates the platform, probing `openat2(2)` support once.
291 #[must_use]
292 pub fn new() -> Self {
293 Self {
294 openat2: sys::openat2_supported(),
295 eintr_retries: Arc::new(AtomicU64::new(0)),
296 eagain_retries: Arc::new(AtomicU64::new(0)),
297 pool: Arc::new(Mutex::new(None)),
298 tier: LinuxTier::Sync,
299 reason: Arc::from("synchronous by default (PosixPlatform::new)"),
300 auto: false,
301 fell_back: Arc::new(AtomicU64::new(0)),
302 refused: Arc::new(AtomicU64::new(0)),
303 registration: Arc::default(),
304 ringed: Arc::new(AtomicU64::new(0)),
305 last_refusal: Arc::new(core::sync::atomic::AtomicI32::new(0)),
306 }
307 }
308
309 /// A platform whose queues use `tier`: [`LinuxTier::Auto`] takes
310 /// io_uring where the kernel accepts store-io's rings of both
311 /// flavours (probed once per process), else the synchronous tier.
312 /// The choice and its reason are [`Platform::io_path`]. Under `Auto`,
313 /// a queue whose ring the kernel refuses later (for want of
314 /// descriptors or memory, or a filter installed since the probe) is
315 /// a synchronous queue instead, counted in the reason at the time it
316 /// is read (and the interface is synchronous if every queue fell
317 /// back); any other refusal, and every refusal under `Uring`, is an
318 /// error.
319 ///
320 /// # Errors
321 ///
322 /// For [`LinuxTier::Uring`] only: the kernel's refusal to set up a
323 /// ring (io_uring disabled, too old, or without a feature store-io
324 /// requires: `ENOSYS`).
325 pub fn with_tier(tier: LinuxTier) -> RawResult<Self> {
326 let auto = !matches!(tier, LinuxTier::Sync | LinuxTier::Uring);
327 let (tier, reason) = match tier {
328 LinuxTier::Sync => (LinuxTier::Sync, "chosen".to_owned()),
329 LinuxTier::Uring => match uring_available() {
330 Ok(()) => (LinuxTier::Uring, "chosen".to_owned()),
331 Err((e, _)) => return Err(e),
332 },
333 _ => match uring_available() {
334 Ok(()) => (
335 LinuxTier::Uring,
336 "auto: the kernel accepts both ring flavours".to_owned(),
337 ),
338 Err((_, why)) => (LinuxTier::Sync, format!("auto: {why}")),
339 },
340 };
341 Ok(Self {
342 tier,
343 reason: Arc::from(reason),
344 auto: auto && tier == LinuxTier::Uring,
345 ..Self::new()
346 })
347 }
348
349 /// The tier this platform's queues use (`Sync` or `Uring`).
350 #[must_use]
351 pub fn tier(&self) -> LinuxTier {
352 self.tier
353 }
354
355 /// Worker threads in the submit pool, or `None` before the first
356 /// parallel queue starts it. Its size is fixed by that first queue
357 /// (`QueueConfig::pool_threads`, at most 256; else one per CPU up to
358 /// 32).
359 #[must_use]
360 pub fn pool_threads(&self) -> Option<usize> {
361 self.pool
362 .lock()
363 .unwrap_or_else(PoisonError::into_inner)
364 .as_ref()
365 .map(|p| p.threads())
366 }
367
368 /// The submit pool, started with `threads` workers (0 = the default)
369 /// if it is not running yet. A running pool is never resized.
370 fn submit_pool(&self, threads: u32) -> RawResult<Arc<SubmitPool>> {
371 let mut slot = self.pool.lock().unwrap_or_else(PoisonError::into_inner);
372 if let Some(p) = slot.as_ref() {
373 return Ok(Arc::clone(p));
374 }
375 let n = match usize::try_from(threads) {
376 Ok(0) | Err(_) => default_threads(),
377 Ok(n) => n,
378 };
379 let p = Arc::new(SubmitPool::start(n)?);
380 *slot = Some(Arc::clone(&p));
381 Ok(p)
382 }
383
384 /// Whether confined opens use `openat2(RESOLVE_BENEATH|RESOLVE_NO_SYMLINKS)`
385 /// (kernel 5.6 and later) rather than `openat(O_NOFOLLOW)`.
386 #[must_use]
387 pub fn openat2_supported(&self) -> bool {
388 self.openat2
389 }
390
391 /// Opens retried after `EINTR` since creation (each retry counted).
392 #[must_use]
393 pub fn eintr_retries(&self) -> u64 {
394 self.eintr_retries.load(Ordering::Relaxed)
395 }
396
397 /// `openat2` calls retried after a `RESOLVE_BENEATH` race (`EAGAIN`)
398 /// since creation.
399 #[must_use]
400 pub fn eagain_retries(&self) -> u64 {
401 self.eagain_retries.load(Ordering::Relaxed)
402 }
403
404 /// Runs an open, retrying `EINTR` (and `EAGAIN` when `openat2` is in
405 /// use) a bounded number of times. Both precede any I/O.
406 pub(crate) fn retry_open<T>(&self, mut open: impl FnMut() -> RawResult<T>) -> RawResult<T> {
407 let mut attempts = 0;
408 loop {
409 match open() {
410 Err(e) if e.code == libc::EINTR && attempts < OPEN_RETRY_LIMIT => {
411 attempts += 1;
412 let _count = self.eintr_retries.fetch_add(1, Ordering::Relaxed);
413 }
414 Err(e)
415 if e.code == libc::EAGAIN
416 && self.openat2
417 && attempts < OPEN_RETRY_LIMIT =>
418 {
419 attempts += 1;
420 let _count = self.eagain_retries.fetch_add(1, Ordering::Relaxed);
421 }
422 r => return r,
423 }
424 }
425 }
426 }
427
428 impl Platform for PosixPlatform {
429 type File = PosixFile;
430 type Dir = PosixDir;
431 type Lock = PosixLock;
432 type Queue = PosixQueue;
433
434 fn open_dir(&self, path: &Path, create: bool) -> RawResult<PosixDir> {
435 PosixDir::open(self, path, create)
436 }
437
438 fn create_file(&self, dir: &PosixDir, name: &FileName) -> RawResult<PosixFile> {
439 PosixFile::create(self, dir, name)
440 }
441
442 fn open_file(
443 &self,
444 dir: &PosixDir,
445 name: &FileName,
446 mode: FileMode,
447 ) -> RawResult<PosixFile> {
448 PosixFile::open(self, dir, name, mode)
449 }
450
451 fn rename_replace(
452 &self,
453 dir: &PosixDir,
454 from: &FileName,
455 to: &FileName,
456 _file: &PosixFile,
457 ) -> RawResult<()> {
458 dir.rename(from, to)
459 }
460
461 fn rename_noreplace(
462 &self,
463 dir: &PosixDir,
464 from: &FileName,
465 to: &FileName,
466 _file: &PosixFile,
467 ) -> RawResult<()> {
468 dir.rename_noreplace(from, to)
469 }
470
471 fn unlink(&self, dir: &PosixDir, name: &FileName) -> RawResult<()> {
472 dir.unlink(name)
473 }
474
475 fn sync_dir(&self, dir: &PosixDir) -> RawResult<()> {
476 dir.sync()
477 }
478
479 fn lock_exclusive(&self, file: &PosixFile) -> RawResult<PosixLock> {
480 PosixLock::acquire(file, false)
481 }
482
483 fn lock_shared(&self, file: &PosixFile) -> RawResult<PosixLock> {
484 PosixLock::acquire(file, true)
485 }
486
487 fn flush_errors_per_handle(&self) -> bool {
488 // errseq_t, per open file description, since Linux 4.13; since
489 // 4.16 a description opened after an unseen error still sees it
490 // (the extra slots' handles are opened after the store's own).
491 sys::kernel_release()
492 .as_deref()
493 .and_then(crate::probe::virt::kernel_version)
494 .is_some_and(|v| v >= (4, 16, 0))
495 }
496
497 fn reopen(&self, file: &PosixFile) -> Option<PosixFile> {
498 file.read_write_twin().ok()
499 }
500
501 fn monotonic_ms(&self) -> u64 {
502 sys::monotonic_coarse_ms()
503 }
504
505 fn io_path(&self) -> IoPath {
506 let fell_back = self.fell_back.load(Ordering::Acquire);
507 let refused = self.refused.load(Ordering::Acquire);
508 let ringed = self.ringed.load(Ordering::Relaxed);
509 let mut reason = self.reason.to_string();
510 let code = self.last_refusal.load(Ordering::Relaxed);
511 if fell_back > 0 {
512 reason.push_str(&format!(
513 "; {fell_back} queue(s) synchronous: the kernel refused a ring ({})",
514 sys::errno(code)
515 ));
516 }
517 let r = &self.registration;
518 let asked = r.asked.load(Ordering::Relaxed);
519 if asked > 0 {
520 reason.push_str(&format!(
521 "; registration (D-078) asked on {asked} ring(s): a direct descriptor on {}",
522 r.files.load(Ordering::Relaxed)
523 ));
524 let refusal = r.refusal.load(Ordering::Relaxed);
525 if refusal != 0 {
526 reason.push_str(&format!(" (refused: {})", sys::errno(refusal)));
527 }
528 }
529 if refused > 0 {
530 reason.push_str(&format!(
531 "; {refused} ring(s) refused by the kernel ({}): a submitter needing one was not created, or a parallel lane's (or an open's header reads') operations ran one at a time",
532 sys::errno(code)
533 ));
534 }
535 IoPath {
536 // Every queue so far fell back: the store is synchronous.
537 interface: if self.tier == LinuxTier::Uring && (ringed > 0 || fell_back == 0) {
538 IoInterface::IoUring
539 } else {
540 IoInterface::Synchronous
541 },
542 reason,
543 }
544 }
545
546 fn allocate(&self, file: &PosixFile, len: u64) -> RawResult<()> {
547 alloc::allocate(file, len)
548 }
549
550 fn allocate_range(&self, file: &PosixFile, offset: u64, len: u64) -> RawResult<()> {
551 alloc::allocate_range(file, offset, len)
552 }
553
554 fn set_len(&self, file: &PosixFile, len: u64) -> RawResult<()> {
555 alloc::set_len(file, len)
556 }
557
558 fn size(&self, file: &PosixFile) -> RawResult<u64> {
559 alloc::size(file)
560 }
561
562 fn release_range(
563 &self,
564 file: &PosixFile,
565 offset: u64,
566 len: u64,
567 how: ReleaseHow,
568 ) -> RawResult<()> {
569 alloc::release(file, offset, len, how)
570 }
571
572 fn range_state(&self, file: &PosixFile, offset: u64, len: u64) -> RawResult<RangeState> {
573 alloc::range_state(file, offset, len)
574 }
575
576 fn flush_all(&self, file: &PosixFile) -> RawResult<()> {
577 sys::fsync(file.raw())
578 }
579
580 fn probe(&self, dir: &PosixDir, file: &PosixFile) -> RawResult<Evidence> {
581 probe::probe(dir, file)
582 }
583
584 fn queue(&self, cfg: QueueConfig) -> RawResult<PosixQueue> {
585 self.make_queue(cfg, None)
586 }
587
588 fn queue_registered(&self, cfg: QueueConfig, file: &PosixFile) -> RawResult<PosixQueue> {
589 // Only a thread-owned ring registers (R3.3): a read-only twin of
590 // the file, opened only once a ring is made (`file::read_only_twin`).
591 let reg = (cfg.owner == Owner::Thread).then(|| crate::uring::queue::Registered {
592 file: file.share_fd(),
593 });
594 self.make_queue(cfg, reg)
595 }
596 }
597
598 impl PosixPlatform {
599 fn make_queue(
600 &self,
601 cfg: QueueConfig,
602 reg: Option<crate::uring::queue::Registered>,
603 ) -> RawResult<PosixQueue> {
604 // Rings for queues that keep several operations in flight (a
605 // `Submitter`, a parallel lane). A queue whose caller runs one
606 // operation at a time and waits for it (`parallel: false`, a
607 // blocking call's lane) does it inline with one system call on
608 // the io_uring tier too: a ring round trip for a lone operation
609 // costs more than the call (D-104), and buys nothing.
610 let ring = cfg.parallel || cfg.owner == Owner::Thread;
611 if self.tier == LinuxTier::Uring && ring {
612 let owned = cfg.owner == Owner::Thread;
613 let asked = reg.is_some();
614 match PosixQueue::uring(cfg, owned, reg) {
615 Ok(q) => {
616 let _n = self.ringed.fetch_add(1, Ordering::Relaxed);
617 if asked {
618 self.count_registration(&q);
619 }
620 return Ok(q);
621 }
622 // `Auto` chose rings; this one was refused: a synchronous
623 // queue, which needs nothing from the kernel, serves
624 // (below), and is counted.
625 Err(e) if self.auto && passes_to_sync(e) => {
626 self.last_refusal.store(e.code, Ordering::Relaxed);
627 let _n = self.fell_back.fetch_add(1, Ordering::Release);
628 }
629 Err(e) => {
630 self.last_refusal.store(e.code, Ordering::Relaxed);
631 let _n = self.refused.fetch_add(1, Ordering::Release);
632 return Err(e);
633 }
634 }
635 }
636 if cfg.parallel {
637 Ok(PosixQueue::pooled(cfg, self.submit_pool(cfg.pool_threads)?))
638 } else {
639 Ok(PosixQueue::new(cfg))
640 }
641 }
642
643 /// Counts what a ring asked to register got (D-078), for `io_path`.
644 fn count_registration(&self, q: &PosixQueue) {
645 let Some(got) = q.registered() else {
646 return;
647 };
648 let r = &self.registration;
649 let _a = r.asked.fetch_add(1, Ordering::Relaxed);
650 if got.file {
651 let _f = r.files.fetch_add(1, Ordering::Relaxed);
652 }
653 if let Some(e) = got.refused {
654 r.refusal.store(e.code, Ordering::Relaxed);
655 }
656 }
657 }
658}
659
660#[cfg(target_os = "linux")]
661pub use platform::PosixPlatform;