Skip to main content

oxidelake_storage/
uring.rs

1//! io_uring-backed local object store (feature `io-uring`, Linux only).
2//!
3//! A dedicated storage thread owns the ring: ranged reads and whole-object
4//! writes are submitted as `IORING_OP_READ` / `IORING_OP_WRITE` and their
5//! completions are bridged back to async callers through `oneshot` channels.
6//! A write goes to a staged file beside the object and is renamed over it,
7//! as `LocalFileSystem` does. A read is opened, and its preconditions and
8//! range checked, by the wrapped [`LocalFileSystem`]'s `get_opts`; the ring
9//! reads from the descriptor that check was made on. Metadata operations
10//! (`list`, `head`, `delete`, `copy`, multipart uploads) delegate to it
11//! outright, so both stores behave identically — the shared conformance
12//! test in `tests/` runs against both.
13//!
14//! Construction fails with a typed error where io_uring is unavailable
15//! (EPERM/ENOSYS in sandboxes, old kernels); callers fall back to
16//! [`LocalFileSystem`].
17
18use std::fmt;
19use std::fs::{File, OpenOptions};
20use std::io;
21use std::ops::Range;
22use std::os::fd::AsRawFd;
23use std::os::unix::fs::MetadataExt;
24use std::path::PathBuf;
25use std::sync::Arc;
26use std::time::UNIX_EPOCH;
27
28use async_trait::async_trait;
29use bytes::Bytes;
30use futures::stream::{self, BoxStream, StreamExt};
31use io_uring::{IoUring, opcode, types};
32use object_store::local::LocalFileSystem;
33use object_store::path::Path;
34use object_store::{
35    Attributes, CopyOptions, Error, GetOptions, GetResult, GetResultPayload, ListResult,
36    MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, PutOptions, PutPayload,
37    PutResult, RenameOptions, Result,
38};
39use oxidelake_core::EngineError;
40use tokio::sync::{mpsc, oneshot};
41
42const STORE: &str = "UringLocalFileSystem";
43
44/// How many jobs may wait on the ring thread before a caller is made to
45/// wait too. The worker handles one SQE at a time, so an unbounded queue
46/// only moves the backlog from the callers into memory, where it is neither
47/// visible nor bounded (#44).
48const JOB_QUEUE: usize = 256;
49
50/// The most of an object a streamed get reads at once (#112). `get_opts`
51/// used to read its whole range into one buffer before returning the
52/// stream, so streaming a 1 GiB object cost 1 GiB, and one larger than the
53/// address space left aborted on the allocation. Each chunk is now read on
54/// the ring when the consumer asks for it. 8 MiB keeps a Parquet column
55/// chunk read through `get_range` to one chunk (and no copy) in the common
56/// case; `LocalFileSystem`'s streams use 8 KiB.
57const STREAM_CHUNK: u64 = 8 << 20;
58
59enum Job {
60    Read {
61        /// The descriptor the caller opened and checked (#113). The job
62        /// holds it open until the read completes, whatever the caller does
63        /// meanwhile.
64        file: Arc<File>,
65        range: Range<u64>,
66        reply: oneshot::Sender<io::Result<Bytes>>,
67    },
68    Write {
69        path: PathBuf,
70        payload: PutPayload,
71        /// The new object's e_tag.
72        reply: oneshot::Sender<io::Result<String>>,
73    },
74}
75
76/// How many times to re-enter the ring after a submit error that is not
77/// `EINTR` before giving up on ever seeing the completion. Only reachable if
78/// the ring itself is broken: this worker keeps at most one SQE in flight and
79/// the completion queue is twice the submission queue, so the conditions that
80/// produce `EBUSY`/`EAGAIN` cannot arise here.
81const REAP_ATTEMPTS: u32 = 1024;
82
83/// Submits one SQE, waits for *its* completion, and returns the raw result.
84///
85/// `seq` is the worker's submission counter; this call takes the next value
86/// and tags the SQE with it, so the completion can be matched to the
87/// submission that produced it rather than assumed.
88///
89/// The contract that matters is that this function does not return while the
90/// kernel may still be writing into the caller's buffer (#27). A signal
91/// delivered to this thread makes `submit_and_wait` return `EINTR` with the
92/// SQE still in flight; returning there dropped the caller's `Vec` under the
93/// kernel and left the completion to be read by the *next* job as its own.
94/// So every exit below reaps first, and the error, if any, is reported after
95/// the completion is in hand rather than instead of it.
96fn submit_one(
97    ring: &mut IoUring,
98    entry: &io_uring::squeue::Entry,
99    seq: &mut u64,
100) -> io::Result<i32> {
101    *seq = seq.wrapping_add(1);
102    let tag = *seq;
103    let entry = entry.clone().user_data(tag);
104
105    // SAFETY: the buffer `entry` points at is owned by the caller and lives
106    // until this function returns, and this function returns only after the
107    // kernel's completion for this exact SQE has been reaped below.
108    unsafe { ring.submission().push(&entry) }
109        .map_err(|e| io::Error::other(format!("io_uring submission queue is full: {e:?}")))?;
110
111    // Past this point the SQE belongs to the kernel.
112    let mut failure: Option<io::Error> = None;
113    let mut attempts = 0u32;
114    let result = loop {
115        if let Err(e) = ring.submit_and_wait(1) {
116            if e.kind() == io::ErrorKind::Interrupted {
117                // A signal, not a failure: the SQE is still in flight and the
118                // completion is still coming. This is the path
119                // `tests::a_signal_during_the_wait_does_not_abandon_the_completion`
120                // below drives (`make test-io-uring`).
121                continue;
122            }
123            if failure.is_none() {
124                failure = Some(e);
125            }
126        }
127        match ring.completion().next() {
128            Some(cqe) if cqe.user_data() == tag => break cqe.result(),
129            Some(cqe) => {
130                // A completion for something else means the ring and this
131                // worker disagree about what is outstanding, so no result
132                // from it can be trusted -- including this one.
133                return Err(io::Error::other(format!(
134                    "io_uring completion carries submission {} where {tag} was expected; \
135                     the ring is out of step with the worker",
136                    cqe.user_data()
137                )));
138            }
139            None => {
140                attempts += 1;
141                assert!(
142                    failure.is_none() || attempts < REAP_ATTEMPTS,
143                    "io_uring: submission {tag} was accepted by the kernel but its completion \
144                     never arrived after {REAP_ATTEMPTS} attempts ({}); returning would free a \
145                     buffer the kernel may still be writing into (#27)",
146                    failure
147                        .as_ref()
148                        .map_or_else(|| "no error".to_owned(), |e| e.to_string()),
149                );
150            }
151        }
152    };
153    if let Some(e) = failure {
154        return Err(e);
155    }
156    if result < 0 {
157        return Err(io::Error::from_raw_os_error(-result));
158    }
159    Ok(result)
160}
161
162fn read_range(
163    ring: &mut IoUring,
164    seq: &mut u64,
165    file: &File,
166    range: Range<u64>,
167) -> io::Result<Bytes> {
168    let len = usize::try_from(range.end.saturating_sub(range.start))
169        .map_err(|_| io::Error::other("range too large"))?;
170    let mut buf = vec![0u8; len];
171    let mut done = 0usize;
172    while done < len {
173        let chunk = u32::try_from(len - done).unwrap_or(u32::MAX);
174        let entry = opcode::Read::new(types::Fd(file.as_raw_fd()), buf[done..].as_mut_ptr(), chunk)
175            .offset(range.start + done as u64)
176            .build();
177        let n = submit_one(ring, &entry, seq)?;
178        if n == 0 {
179            return Err(io::Error::new(
180                io::ErrorKind::UnexpectedEof,
181                format!(
182                    "short read: {done} of {len} bytes at offset {}",
183                    range.start
184                ),
185            ));
186        }
187        done += usize::try_from(n).unwrap_or(0);
188    }
189    Ok(Bytes::from(buf))
190}
191
192fn write_at(
193    ring: &mut IoUring,
194    seq: &mut u64,
195    file: &File,
196    data: &[u8],
197    offset: u64,
198) -> io::Result<()> {
199    let mut done = 0usize;
200    while done < data.len() {
201        let chunk = u32::try_from(data.len() - done).unwrap_or(u32::MAX);
202        let entry = opcode::Write::new(types::Fd(file.as_raw_fd()), data[done..].as_ptr(), chunk)
203            .offset(offset + done as u64)
204            .build();
205        let n = submit_one(ring, &entry, seq)?;
206        if n == 0 {
207            return Err(io::Error::new(
208                io::ErrorKind::WriteZero,
209                "io_uring wrote zero bytes",
210            ));
211        }
212        done += usize::try_from(n).unwrap_or(0);
213    }
214    Ok(())
215}
216
217/// The e_tag `LocalFileSystem` reports for a file with this metadata: inode,
218/// modification time in microseconds and size, in hex (object_store 0.13's
219/// private `get_etag`). `head` and `get_opts` answer through
220/// `LocalFileSystem`, so a put must return this exact string for a caller's
221/// `if_match` to work; `a_put_returns_the_e_tag_head_reports` holds the two
222/// together.
223fn e_tag(metadata: &std::fs::Metadata) -> String {
224    let mtime = metadata
225        .modified()
226        .ok()
227        .and_then(|mtime| mtime.duration_since(UNIX_EPOCH).ok())
228        .unwrap_or_default()
229        .as_micros();
230    format!("{:x}-{mtime:x}-{:x}", metadata.ino(), metadata.len())
231}
232
233/// Creates `<path>#<n>` for the first free `n`: the name `LocalFileSystem`
234/// stages an upload under, which its listing skips, so a half-written
235/// object never shows up in `list` whichever store is writing it.
236fn new_staged_upload(path: &std::path::Path) -> io::Result<(File, PathBuf)> {
237    if let Some(parent) = path.parent() {
238        std::fs::create_dir_all(parent)?;
239    }
240    let mut n = 1u64;
241    loop {
242        let mut staged = path.as_os_str().to_owned();
243        staged.push(format!("#{n}"));
244        let staged = PathBuf::from(staged);
245        // `create_new` (O_EXCL) neither follows a symlink planted under the
246        // name nor opens a file another upload is still writing.
247        match OpenOptions::new()
248            .read(true)
249            .write(true)
250            .create_new(true)
251            .open(&staged)
252        {
253            Ok(file) => return Ok((file, staged)),
254            Err(e) if e.kind() == io::ErrorKind::AlreadyExists => n += 1,
255            Err(e) => return Err(e),
256        }
257    }
258}
259
260/// Writes `payload` to a staged file beside `path` and renames it over
261/// `path`, as `LocalFileSystem` does, returning the new object's e_tag
262/// (#111).
263///
264/// The store used to truncate `path` and write it in place. A write that
265/// failed part way (a full disk, a file size limit) then left the first part
266/// of the new object where the old one had been, a reader could see a
267/// half-written object, and a symlink at `path` was written through, to
268/// wherever it pointed, outside the store's root. The rename replaces the
269/// directory entry instead: the old object stays whole until the new one
270/// is, and a link is replaced rather than followed.
271fn write_staged(
272    ring: &mut IoUring,
273    seq: &mut u64,
274    path: &std::path::Path,
275    payload: &PutPayload,
276) -> io::Result<String> {
277    let (file, staged) = new_staged_upload(path)?;
278    let written = publish_staged(ring, seq, file, &staged, path, payload);
279    if written.is_err() {
280        // Best effort, as `LocalFileSystem` does: the object at `path` is
281        // untouched either way.
282        let _ = std::fs::remove_file(&staged);
283    }
284    written
285}
286
287fn publish_staged(
288    ring: &mut IoUring,
289    seq: &mut u64,
290    file: File,
291    staged: &std::path::Path,
292    path: &std::path::Path,
293    payload: &PutPayload,
294) -> io::Result<String> {
295    let mut offset = 0u64;
296    for part in payload.iter() {
297        write_at(ring, seq, &file, part, offset)?;
298        offset += part.len() as u64;
299    }
300    // Taken from the descriptor before the rename, as `LocalFileSystem` does,
301    // so it is this write's e_tag even if another put replaces the object
302    // the moment it lands. The rename keeps inode, mtime and size.
303    let tag = e_tag(&file.metadata()?);
304    // Closed first, as `LocalFileSystem` does: some FUSE filesystems only
305    // publish a file's contents when it is closed.
306    drop(file);
307    std::fs::rename(staged, path)?;
308    Ok(tag)
309}
310
311fn worker(mut rx: mpsc::Receiver<Job>, mut ring: IoUring) {
312    // Submission counter for the ring this thread owns. Every SQE is tagged
313    // with the next value so its completion can be recognised rather than
314    // assumed to be the only one outstanding (#27).
315    let mut seq = 0u64;
316    while let Some(job) = rx.blocking_recv() {
317        match job {
318            Job::Read { file, range, reply } => {
319                let _ = reply.send(read_range(&mut ring, &mut seq, &file, range));
320            }
321            Job::Write {
322                path,
323                payload,
324                reply,
325            } => {
326                let _ = reply.send(write_staged(&mut ring, &mut seq, &path, &payload));
327            }
328        }
329    }
330}
331
332/// Reads `range` of `file` on the ring thread behind `tx`.
333async fn ring_read(
334    tx: &mpsc::Sender<Job>,
335    location: &Path,
336    file: Arc<File>,
337    range: Range<u64>,
338) -> Result<Bytes> {
339    let (reply, rx) = oneshot::channel();
340    tx.send(Job::Read { file, range, reply })
341        .await
342        .map_err(|_| Error::Generic {
343            store: STORE,
344            source: "io_uring worker thread has exited".into(),
345        })?;
346    let result = rx.await.map_err(|_| Error::Generic {
347        store: STORE,
348        source: "io_uring worker dropped the reply".into(),
349    })?;
350    result.map_err(|e| UringLocalFileSystem::io_error(location, e))
351}
352
353/// An object `LocalFileSystem::get_opts` opened and checked: the descriptor
354/// and its answer for it (#113).
355struct Opened {
356    file: Arc<File>,
357    meta: ObjectMeta,
358    range: Range<u64>,
359    attributes: Attributes,
360}
361
362/// A local object store whose data-path reads and writes go through io_uring.
363pub struct UringLocalFileSystem {
364    inner: Arc<LocalFileSystem>,
365    tx: mpsc::Sender<Job>,
366}
367
368impl UringLocalFileSystem {
369    /// Creates the store around `inner`, spawning the ring thread with
370    /// `entries` submission slots. Fails when io_uring is unavailable.
371    pub fn try_new(inner: LocalFileSystem, entries: u32) -> std::result::Result<Self, EngineError> {
372        let ring = IoUring::new(entries.max(2)).map_err(|e| {
373            EngineError::Io(io::Error::new(
374                e.kind(),
375                format!("io_uring is unavailable on this machine ({e}); use LocalFileSystem"),
376            ))
377        })?;
378        let (tx, rx) = mpsc::channel(JOB_QUEUE);
379        std::thread::Builder::new()
380            .name("oxide-io-uring".into())
381            .spawn(move || worker(rx, ring))?;
382        Ok(Self {
383            inner: Arc::new(inner),
384            tx,
385        })
386    }
387
388    /// `Err` with the reason when io_uring cannot be set up here.
389    pub fn probe() -> io::Result<()> {
390        IoUring::new(2).map(|_| ())
391    }
392
393    fn filesystem_path(&self, location: &Path) -> Result<PathBuf> {
394        self.inner.path_to_filesystem(location)
395    }
396
397    fn io_error(location: &Path, err: io::Error) -> Error {
398        if err.kind() == io::ErrorKind::NotFound {
399            Error::NotFound {
400                path: location.to_string(),
401                source: Box::new(err),
402            }
403        } else {
404            Error::Generic {
405                store: STORE,
406                source: Box::new(err),
407            }
408        }
409    }
410
411    /// Opens the object through `LocalFileSystem::get_opts`, which opens the
412    /// file once, checks `options`' preconditions against `fstat` of that
413    /// descriptor and resolves the range against its size, and returns the
414    /// descriptor with its answer (#113).
415    ///
416    /// Everything this store then reads, it reads from that descriptor.
417    /// It used to stat the path for the check and have the worker open the
418    /// path again to read, so an object replaced in between came back as the
419    /// new bytes under the old object's size and e_tag. Taking the check
420    /// from `LocalFileSystem` also keeps its errors and its e_tags, which
421    /// is what callers of either store see.
422    async fn open(&self, location: &Path, options: GetOptions) -> Result<Opened> {
423        let GetResult {
424            payload,
425            meta,
426            range,
427            attributes,
428        } = self.inner.get_opts(location, options).await?;
429        match payload {
430            GetResultPayload::File(file, _) => Ok(Opened {
431                file: Arc::new(file),
432                meta,
433                range,
434                attributes,
435            }),
436            GetResultPayload::Stream(_) => Err(Error::Generic {
437                store: STORE,
438                source: "LocalFileSystem answered a get with a stream rather than a file".into(),
439            }),
440        }
441    }
442
443    async fn read(&self, location: &Path, file: Arc<File>, range: Range<u64>) -> Result<Bytes> {
444        ring_read(&self.tx, location, file, range).await
445    }
446
447    /// `range` of `file` as a stream of chunks of at most [`STREAM_CHUNK`]
448    /// bytes, each read on the ring only when the consumer pulls it (#112):
449    /// the stream holds one chunk at a time, whatever the object's size.
450    fn chunks(
451        &self,
452        location: &Path,
453        file: Arc<File>,
454        range: Range<u64>,
455    ) -> BoxStream<'static, Result<Bytes>> {
456        let tx = self.tx.clone();
457        let location = location.clone();
458        let end = range.end;
459        stream::try_unfold(range.start, move |offset| {
460            let (tx, location, file) = (tx.clone(), location.clone(), Arc::clone(&file));
461            async move {
462                if offset >= end {
463                    return Ok(None);
464                }
465                let next = end.min(offset.saturating_add(STREAM_CHUNK));
466                let chunk = ring_read(&tx, &location, file, offset..next).await?;
467                Ok(Some((chunk, next)))
468            }
469        })
470        .boxed()
471    }
472
473    /// Writes `payload` as the object at `location` and returns its e_tag.
474    async fn write(&self, location: &Path, payload: PutPayload) -> Result<String> {
475        let path = self.filesystem_path(location)?;
476        let (reply, rx) = oneshot::channel();
477        self.tx
478            .send(Job::Write {
479                path,
480                payload,
481                reply,
482            })
483            .await
484            .map_err(|_| Error::Generic {
485                store: STORE,
486                source: "io_uring worker thread has exited".into(),
487            })?;
488        let result = rx.await.map_err(|_| Error::Generic {
489            store: STORE,
490            source: "io_uring worker dropped the reply".into(),
491        })?;
492        result.map_err(|e| Self::io_error(location, e))
493    }
494}
495
496impl fmt::Debug for UringLocalFileSystem {
497    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
498        f.debug_struct(STORE).field("inner", &self.inner).finish()
499    }
500}
501
502impl fmt::Display for UringLocalFileSystem {
503    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
504        write!(f, "{STORE}({})", self.inner)
505    }
506}
507
508#[async_trait]
509impl ObjectStore for UringLocalFileSystem {
510    async fn put_opts(
511        &self,
512        location: &Path,
513        payload: PutPayload,
514        opts: PutOptions,
515    ) -> Result<PutResult> {
516        match opts.mode {
517            PutMode::Overwrite => {
518                // Refused the way `LocalFileSystem` refuses them: accepting
519                // attributes and then dropping them told the caller they
520                // were stored (#111).
521                if !opts.attributes.is_empty() {
522                    return Err(Error::NotImplemented {
523                        operation: "`put_opts` with `opts.attributes` specified".into(),
524                        implementer: self.to_string(),
525                    });
526                }
527                let e_tag = self.write(location, payload).await?;
528                Ok(PutResult {
529                    e_tag: Some(e_tag),
530                    version: None,
531                })
532            }
533            // `Create` and `Update` stay `LocalFileSystem`'s, refusals and
534            // the order it checks mode and attributes in included.
535            _ => self.inner.put_opts(location, payload, opts).await,
536        }
537    }
538
539    async fn put_multipart_opts(
540        &self,
541        location: &Path,
542        opts: PutMultipartOptions,
543    ) -> Result<Box<dyn MultipartUpload>> {
544        self.inner.put_multipart_opts(location, opts).await
545    }
546
547    async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
548        // Preconditions (#44) and the range are `LocalFileSystem`'s answer
549        // for the descriptor `open` returns, and the stream below reads from
550        // that descriptor (#113), a bounded chunk at a time (#112).
551        let head = options.head;
552        let Opened {
553            file,
554            meta,
555            range,
556            attributes,
557        } = self.open(location, options).await?;
558        let payload = if head {
559            stream::empty().boxed()
560        } else {
561            self.chunks(location, file, range.clone())
562        };
563        Ok(GetResult {
564            payload: GetResultPayload::Stream(payload),
565            meta,
566            range,
567            attributes,
568        })
569    }
570
571    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
572        // One descriptor for every range, as `LocalFileSystem` reads them:
573        // an object replaced part way through comes back from one version,
574        // not from two (#113).
575        let Opened { file, .. } = self.open(location, GetOptions::default()).await?;
576        let mut out = Vec::with_capacity(ranges.len());
577        for range in ranges {
578            out.push(
579                self.read(location, Arc::clone(&file), range.clone())
580                    .await?,
581            );
582        }
583        Ok(out)
584    }
585
586    fn delete_stream(
587        &self,
588        locations: BoxStream<'static, Result<Path>>,
589    ) -> BoxStream<'static, Result<Path>> {
590        self.inner.delete_stream(locations)
591    }
592
593    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
594        self.inner.list(prefix)
595    }
596
597    fn list_with_offset(
598        &self,
599        prefix: Option<&Path>,
600        offset: &Path,
601    ) -> BoxStream<'static, Result<ObjectMeta>> {
602        self.inner.list_with_offset(prefix, offset)
603    }
604
605    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
606        self.inner.list_with_delimiter(prefix).await
607    }
608
609    async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
610        self.inner.copy_opts(from, to, options).await
611    }
612
613    async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
614        self.inner.rename_opts(from, to, options).await
615    }
616}
617
618#[cfg(test)]
619#[allow(clippy::unwrap_used, clippy::expect_used)]
620mod tests {
621    use super::*;
622
623    /// A handler that does nothing, installed *without* `SA_RESTART` so the
624    /// interrupted syscall really returns `EINTR` rather than being restarted
625    /// by the kernel. Both halves matter: with no handler at all `SIGUSR1`
626    /// terminates the process, and with `SA_RESTART` the test would pass
627    /// against the very bug it exists to catch.
628    extern "C" fn noop(_: libc::c_int) {}
629
630    fn install_interrupting_handler() {
631        // SAFETY: a plain extern "C" handler with an empty mask; `noop`
632        // touches nothing, so it is async-signal-safe.
633        unsafe {
634            let mut action: libc::sigaction = std::mem::zeroed();
635            action.sa_sigaction = noop as *const () as usize;
636            action.sa_flags = 0;
637            libc::sigemptyset(&mut action.sa_mask);
638            assert_eq!(
639                libc::sigaction(libc::SIGUSR1, &action, std::ptr::null_mut()),
640                0,
641                "could not install the SIGUSR1 handler: {}",
642                io::Error::last_os_error()
643            );
644        }
645    }
646
647    /// The regression test for #27, aimed rather than sprayed.
648    ///
649    /// An end-to-end read is the wrong instrument: a local read completes in
650    /// microseconds, so a signaller racing it almost never lands inside
651    /// `submit_and_wait`, and such a test passes against the bug. This drives
652    /// the path directly instead — a read submitted against an *empty pipe*,
653    /// which cannot complete until somebody writes, so the worker is
654    /// guaranteed to be blocked in `submit_and_wait` when the signal arrives.
655    /// `tgkill` targets this exact thread rather than the process.
656    ///
657    /// Before the fix `submit_one` returned `Err(EINTR)` here, leaving the
658    /// SQE in flight and `buf` to be dropped under the kernel. Now the signal
659    /// is invisible: the read completes and its bytes arrive.
660    #[test]
661    fn a_signal_during_the_wait_does_not_abandon_the_completion() {
662        let Ok(mut ring) = IoUring::new(4) else {
663            eprintln!("SKIPPED: io_uring unavailable in this environment");
664            return;
665        };
666        install_interrupting_handler();
667
668        // SAFETY: a pipe pair; both descriptors are closed at the end.
669        let mut fds = [0 as libc::c_int; 2];
670        assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0);
671        let (read_fd, write_fd) = (fds[0], fds[1]);
672
673        // SAFETY: `gettid` and `getpid` take no arguments and only read.
674        let tid = unsafe { libc::syscall(libc::SYS_gettid) } as libc::pid_t;
675        let pid = unsafe { libc::getpid() };
676
677        let signaller = std::thread::spawn(move || {
678            // Long enough that the read is certainly parked in the kernel.
679            std::thread::sleep(std::time::Duration::from_millis(50));
680            for _ in 0..5 {
681                // SAFETY: signalling a thread of our own process.
682                unsafe { libc::syscall(libc::SYS_tgkill, pid, tid, libc::SIGUSR1) };
683                std::thread::sleep(std::time::Duration::from_millis(10));
684            }
685            // Only now can the read complete, so every signal above landed
686            // while the ring was waiting.
687            let payload = b"uring-ok";
688            // SAFETY: writing 8 bytes from a live buffer to the pipe.
689            unsafe { libc::write(write_fd, payload.as_ptr().cast(), payload.len()) };
690        });
691
692        let mut buf = [0u8; 8];
693        let entry = opcode::Read::new(types::Fd(read_fd), buf.as_mut_ptr(), 8).build();
694        let mut seq = 0u64;
695        let n = submit_one(&mut ring, &entry, &mut seq)
696            .expect("a signal during the wait is not a failure: the SQE is still in flight");
697
698        signaller.join().unwrap();
699        assert_eq!(n, 8, "the whole payload was read");
700        assert_eq!(&buf, b"uring-ok", "the bytes are the ones written");
701        assert_eq!(seq, 1, "the submission was tagged");
702
703        // SAFETY: closing descriptors this test opened and no longer uses.
704        unsafe {
705            libc::close(read_fd);
706            libc::close(write_fd);
707        }
708    }
709
710    /// The tag is what makes a completion recognisable as *this* submission's
711    /// (#27). Two submissions in a row must carry different ones, or a stale
712    /// completion is indistinguishable from the expected one.
713    #[test]
714    fn each_submission_is_tagged_with_a_fresh_value() {
715        let Ok(mut ring) = IoUring::new(4) else {
716            eprintln!("SKIPPED: io_uring unavailable in this environment");
717            return;
718        };
719        let dir = tempfile::tempdir().unwrap();
720        let path = dir.path().join("tagged.bin");
721        std::fs::write(&path, b"0123456789").unwrap();
722        let file = File::open(&path).unwrap();
723
724        let mut seq = 0u64;
725        for expected in 1..=3u64 {
726            let mut buf = [0u8; 10];
727            let entry =
728                opcode::Read::new(types::Fd(file.as_raw_fd()), buf.as_mut_ptr(), 10).build();
729            let n = submit_one(&mut ring, &entry, &mut seq).unwrap();
730            assert_eq!(n, 10);
731            assert_eq!(&buf, b"0123456789");
732            assert_eq!(seq, expected, "each submission takes the next tag");
733        }
734    }
735}