Skip to main content

store_io_engine/submitter/
mod.rs

1//! The engine-layer [`Submitter`] (D-065, D-080, D-098): many operations in
2//! flight from one thread, without blocking, with the same integrity rules
3//! as the blocking calls.
4//!
5//! Every operation is admitted like a blocking call (the region's gate, the
6//! store's state, the same validation), then held in a fixed table until its
7//! completion is processed by [`Submitter::reap`] or [`Submitter::wait`]:
8//!
9//! - its region's **owned pass**, so recycle and release wait for it;
10//! - for an append, its **frontier reservation**, completed only when the
11//!   write transferred every byte (else the store is poisoned: the reserved
12//!   range is a hole no later append may be acknowledged past);
13//! - its tag and submission sequence.
14//!
15//! A [`WriteTicket`](crate::WriteTicket) is minted only from a full
16//! completion (D-080). Queue tags are table indices plus a generation, so a
17//! completion matches exactly one operation. Nothing allocates after
18//! creation. Durable writes and barriers ([`durable`]) wait for a flush
19//! without blocking: the store's flush thread answers them (D-099).
20
21mod durable;
22
23use std::marker::PhantomData;
24use std::sync::Arc;
25
26use store_io_core::error::{Error, ErrorContext, FirstCause, NotWrittenCause, Op, OsError};
27use store_io_core::observe::Counter;
28use store_io_platform::{IoBuf, IoOp, Owner, Platform, Queue, QueueConfig};
29
30use crate::completion::{Completions, Rejected, SubmitterConfig, WriteMode};
31use crate::exec::LaneMark;
32use crate::flusher::{Inbox, SubmitterWaker, WakerOf};
33use crate::gate::{Gate, Gated, OwnedPass};
34use crate::io::{ctx, ctx_range, not_written};
35use crate::lifecycle::retired_error;
36use crate::optable::{Finish, Kind, Split};
37use crate::queue_core::Core;
38use crate::receipt::RegionPos;
39use crate::region::{AppendRegion, PageRegion};
40use crate::store::{LiveRegion, Store};
41
42/// The largest `SubmitterConfig::depth`.
43pub const MAX_DEPTH: u32 = 4096;
44
45impl Gated for LiveRegion {
46    fn gate(&self) -> &Gate {
47        &self.gate
48    }
49}
50
51/// Many reads and writes in flight from one thread (D-065, D-098).
52///
53/// Submitting never blocks: a full submitter is refused with the buffer
54/// given back ([`Rejected`], `NotWritten(QueueFull)`). Completions are
55/// handed back by [`Self::reap`] (never blocks) and [`Self::wait`]
56/// (blocks), with a [`WriteTicket`](crate::WriteTicket) for every write that
57/// completed in full. Buffers come from [`Self::take_buf`] (the store's
58/// pool, shared with its blocking calls: buffers held in flight or kept by
59/// the caller are not available to them) and come back in the completion.
60///
61/// The rules the blocking calls keep hold per operation:
62///
63/// - **An append's position is final at submission** (returned at once),
64///   but it counts as written only once its completion is processed by
65///   this submitter's `reap` or `wait`. Until then, on that region, from
66///   any thread: `sync_through` of any later append waits for it, and an
67///   append starting more than `StoreOptions::append_window` bytes past it
68///   waits (blocking calls) or is refused `NotWritten(TooManyInFlight)`
69///   (submitters). Reap your own appends before syncing or appending far
70///   past them, or the thread waits for itself.
71/// - **Recycle and release wait** for every operation of the submitter on
72///   that region until its completion is processed: a thread must reap its
73///   own submitter's operations on a region before it recycles or releases
74///   that region.
75/// - **A write that fails or transfers short after it was issued** is
76///   `DurabilityUnknown`, and the store is poisoned.
77///
78/// Dropping the submitter waits, untimed, for every operation in flight
79/// before it releases anything the device may still touch. A device that
80/// never completes an operation blocks the drop: the fail-stop choice
81/// (see `Store::oldest_outstanding` for a watchdog).
82///
83/// A submitter belongs to the thread that created it:
84///
85/// ```compile_fail,E0277
86/// fn need_send<T: Send>() {}
87/// need_send::<store_io_engine::Submitter<store_io_sim::SimPlatform>>();
88/// ```
89pub struct Submitter<P: Platform>
90where
91    P::Queue: Send,
92{
93    // Declared first, so its queue drops (after `Drop::drop` drained it)
94    // before the table releases the passes and reservations.
95    core: Core<P::Queue>,
96    /// Where the flush thread answers this submitter's flush requests.
97    inbox: Arc<Inbox<<P::Queue as Queue>::Waker>>,
98    id: u64,
99    store: Store<P>,
100    _thread: PhantomData<*const ()>,
101}
102
103impl<P: Platform> core::fmt::Debug for Submitter<P>
104where
105    P::Queue: Send,
106{
107    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
108        f.debug_struct("Submitter")
109            .field("depth", &self.core.table.depth())
110            .field("in_flight", &self.core.table.live)
111            .field("ready", &self.core.table.ready.len())
112            .finish()
113    }
114}
115
116/// The request [`Submitter::issue`] turns into an [`IoOp`].
117enum Req {
118    Write { offset: u64, buf: IoBuf },
119    Read { offset: u64, buf: IoBuf },
120}
121
122/// The first device operation of `[start, start + buf.len())`: all of it,
123/// or (format 2, across a run's end, spec ยง11) the piece up to the run's
124/// end, `buf` shortened to it, the rest following under the same row
125/// ([`Split`]). Returns its file offset.
126fn first_piece(state: &LiveRegion, start: u64, buf: &mut IoBuf) -> (u64, Option<Split>) {
127    let total = buf.len();
128    let piece = (state.map.run_left(start) as usize).min(total);
129    let split = (piece < total).then(|| {
130        let fits = buf.set_len(piece);
131        debug_assert!(fits, "a piece longer than its buffer");
132        Split { total, done: 0 }
133    });
134    (state.map.locate(start, piece as u64), split)
135}
136
137fn rejected(error: Error, buf: IoBuf) -> Rejected {
138    Rejected {
139        error,
140        buf: Some(buf),
141    }
142}
143
144/// `?` for admission: refuses with the buffer given back.
145macro_rules! admit {
146    ($e:expr, $buf:ident) => {
147        match $e {
148            Ok(v) => v,
149            Err(error) => return Err(rejected(error, $buf)),
150        }
151    };
152}
153
154impl<P: Platform> Store<P>
155where
156    P::Queue: Send,
157{
158    /// A [`Submitter`] for this thread, with its own queue of
159    /// `cfg.depth` operations (1 to [`MAX_DEPTH`]).
160    ///
161    /// # Errors
162    ///
163    /// `Io` if the platform could not create the queue.
164    pub fn submitter(&self, cfg: SubmitterConfig) -> Result<Submitter<P>, Error> {
165        let depth = cfg.depth.clamp(1, MAX_DEPTH);
166        let qcfg = QueueConfig {
167            depth,
168            parallel: true,
169            pool_threads: self.inner.opts.t3_pool_threads,
170            owner: Owner::Thread,
171        };
172        let inner = &*self.inner;
173        // The file outlives the queue: a registering backend keeps its own
174        // reference to it (`Platform::queue_registered`).
175        let queue = if inner.opts.register_file {
176            inner.platform.queue_registered(qcfg, &inner.file)
177        } else {
178            inner.platform.queue(qcfg)
179        };
180        let queue = queue.map_err(|raw| Error::Io {
181            op: Op::Queue,
182            raw,
183            ctx: ErrorContext::default(),
184        })?;
185        let n = depth as usize;
186        let mark = Arc::new(LaneMark::new());
187        let inbox = Arc::new(Inbox::new(n, queue.waker()));
188        let id = self.register_submitter(&mark);
189        Ok(Submitter {
190            core: Core::new(queue, n, mark),
191            inbox,
192            id,
193            store: self.clone(),
194            _thread: PhantomData,
195        })
196    }
197}
198
199impl<P: Platform> Submitter<P>
200where
201    P::Queue: Send,
202{
203    /// A buffer of `len` bytes from the store's pool, for one operation.
204    ///
205    /// # Errors
206    ///
207    /// `NotWritten(TooLarge)` beyond [`Self::max_buf`];
208    /// `NotWritten(PoolExhausted)` if none is free.
209    pub fn take_buf(&self, len: usize) -> Result<IoBuf, Error> {
210        if len > self.max_buf() {
211            return Err(not_written(
212                NotWrittenCause::TooLarge,
213                ErrorContext::default(),
214            ));
215        }
216        self.store.take_any(len, ErrorContext::default())
217    }
218
219    /// The largest buffer, and so the largest single operation, in bytes (a
220    /// whole number of blocks): the pool's largest class, `max_write`
221    /// rounded up to a power of two (or `max_io` without large buffers). One
222    /// OS operation (D-101); the kernel may still split it into several
223    /// device requests.
224    #[must_use]
225    pub fn max_buf(&self) -> usize {
226        self.store.max_op()
227    }
228
229    /// Operations submitted whose completions have not been handed to
230    /// `reap` or `wait` yet: I/O in flight, and durable writes and barriers
231    /// waiting for their flush.
232    #[must_use]
233    pub fn in_flight(&self) -> usize {
234        // Never more than are really pending (an operation lost to a panic
235        // in processing is no longer waited for; the store is poisoned then).
236        self.core
237            .table
238            .live
239            .min(self.core.queue.in_flight() + self.core.table.asked)
240    }
241
242    /// Ends this submitter's current or next [`Self::wait`] from another
243    /// thread (D-083), whether it waits for I/O or for a flush.
244    #[must_use]
245    pub fn waker(&self) -> SubmitterWaker<WakerOf<P>> {
246        SubmitterWaker::new(&self.inbox)
247    }
248
249    /// The checks every operation makes first: this store's region, room in
250    /// the table, and the region's gate.
251    fn admit(
252        &self,
253        store: &Store<P>,
254        state: &Arc<LiveRegion>,
255        buf: &IoBuf,
256    ) -> Result<OwnedPass<LiveRegion>, Error> {
257        let c = ctx(state);
258        if !Arc::ptr_eq(&store.inner, &self.store.inner) {
259            return Err(not_written(NotWrittenCause::ForeignPosition, c));
260        }
261        // Only this store's buffers have its alignment and whole blocks of
262        // capacity; any other could fail at the device after an append's
263        // space is reserved.
264        if !self.store.inner.pool.owns(buf) {
265            return Err(not_written(NotWrittenCause::ForeignBuffer, c));
266        }
267        if self.core.table.full() {
268            return Err(not_written(NotWrittenCause::QueueFull, c));
269        }
270        OwnedPass::enter(state).map_err(|why| retired_error(why, state))
271    }
272
273    /// Appends the buffer's bytes (any length of at least one byte) and
274    /// returns their position at once; the completion carries the ticket
275    /// (and, with [`WriteMode::Durable`], the receipt).
276    /// The append starts at a block boundary and the buffer's tail is
277    /// zero-padded in place to the next one (no copy): the buffer comes back
278    /// with its length set to the padded size.
279    ///
280    /// # Errors
281    ///
282    /// [`Rejected`], with the buffer: `NotWritten` (`QueueFull`: the
283    /// submitter is full; `TooManyInFlight`: the region's append ring is
284    /// full, see [`Submitter`]; empty, region full, read-only, not
285    /// positioned, another store's region or buffer), `Poisoned`, or
286    /// `DurabilityUnknown` if the OS refused the write after its position
287    /// was reserved (the store is then poisoned, as for a blocking append).
288    pub fn append(
289        &mut self,
290        region: &AppendRegion<P>,
291        mut buf: IoBuf,
292        mode: WriteMode,
293        tag: u64,
294    ) -> Result<RegionPos, Rejected> {
295        let state = region.state();
296        let c = ctx(state);
297        let pass = admit!(self.admit(region.store(), state, &buf), buf);
298        admit!(self.store.ready_for(state), buf);
299        let Some(f) = state.frontier.as_ref() else {
300            return Err(rejected(not_written(NotWrittenCause::NotReady, c), buf));
301        };
302        if !state.positioned.load(std::sync::atomic::Ordering::Acquire) {
303            return Err(rejected(
304                not_written(NotWrittenCause::NotPositioned, c),
305                buf,
306            ));
307        }
308        let len = buf.len();
309        if len == 0 {
310            return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
311        }
312        // Checked before the reservation, so the padding below cannot fail
313        // (the pool's capacities are whole blocks anyway).
314        let block = self.store.block() as usize;
315        if len.div_ceil(block) * block > buf.capacity() {
316            return Err(rejected(not_written(NotWrittenCause::TooLarge, c), buf));
317        }
318        let domain = &self.store.inner.domain;
319        let reserved = f
320            .reserve(len as u64)
321            .map_err(|why| match domain.poisoned() {
322                Some(first) => Error::Poisoned { first },
323                None => not_written(why, c),
324            });
325        let r = admit!(reserved, buf);
326        // Held until the row holds it: a panic before then (in `locate`, or
327        // a backend's `submit`) poisons and wakes every waiter at once,
328        // never a hole that a later barrier waits on until this submitter
329        // drops. The store is the region's (`admit` checked it is this one).
330        let held = region.store().hold_reservation(r);
331        let padded = (r.end - r.start) as usize;
332        buf.as_mut_capacity()[len..padded].fill(0);
333        let _fits = buf.set_len(padded);
334        let pos = RegionPos {
335            volume: self.store.inner.volume,
336            region: state.id,
337            generation: state.generation,
338            offset: r.start,
339        };
340        let (offset, split) = first_piece(state, r.start, &mut buf);
341        let req = Req::Write { offset, buf };
342        let durable = mode == WriteMode::Durable;
343        let issued = self.issue(req, tag, Kind::Append(r), pass, durable, split);
344        let _r = held.disarm();
345        match issued {
346            Ok(()) => Ok(pos),
347            // Past the reservation: a hole, as for a blocking append.
348            Err((raw, buf)) => Err(Rejected {
349                error: self
350                    .store
351                    .unknown(Op::Write, raw, ctx_range(state, r.start, r.data_end)),
352                buf,
353            }),
354        }
355    }
356
357    /// Writes the buffer (whole blocks) at `pos`; the completion carries the
358    /// ticket (and, with [`WriteMode::Durable`], the receipt).
359    ///
360    /// # Errors
361    ///
362    /// [`Rejected`], with the buffer: `NotWritten` (foreign, stale,
363    /// misaligned, out of range, submitter full, read-only, another store's
364    /// buffer), `Poisoned`, or `Io` if the OS refused it (nothing reached
365    /// the device).
366    pub fn write(
367        &mut self,
368        region: &PageRegion<P>,
369        pos: RegionPos,
370        mut buf: IoBuf,
371        mode: WriteMode,
372        tag: u64,
373    ) -> Result<(), Rejected> {
374        let state = region.state();
375        let pass = admit!(self.admit(region.store(), state, &buf), buf);
376        admit!(self.store.ready_for(state), buf);
377        let end = admit!(region.check(&pos, buf.len()), buf);
378        let c = ctx_range(state, pos.offset, end);
379        let (offset, split) = first_piece(state, pos.offset, &mut buf);
380        let req = Req::Write { offset, buf };
381        let kind = Kind::Page {
382            start: pos.offset,
383            end,
384        };
385        self.issue(req, tag, kind, pass, mode == WriteMode::Durable, split)
386            .map_err(|(raw, buf)| Rejected {
387                error: Store::<P>::refused(raw, c),
388                buf,
389            })
390    }
391
392    /// Reads into the buffer (whole blocks) from the block-aligned `offset`
393    /// of an append region; the completion carries the bytes. Reads work on
394    /// read-only and poisoned stores.
395    ///
396    /// # Errors
397    ///
398    /// [`Rejected`], with the buffer: `NotWritten` (empty, misaligned, out
399    /// of range, submitter full, another store's region or buffer), or `Io`
400    /// if the OS refused it.
401    pub fn read_append(
402        &mut self,
403        region: &AppendRegion<P>,
404        offset: u64,
405        buf: IoBuf,
406        tag: u64,
407    ) -> Result<(), Rejected> {
408        self.read(region.store(), region.state(), offset, buf, tag)
409    }
410
411    /// Reads into the buffer (whole blocks) at `pos` of a page region; the
412    /// completion carries the bytes.
413    ///
414    /// # Errors
415    ///
416    /// As [`Self::read_append`], and `NotWritten(ForeignPosition |
417    /// StaleGeneration)` for a position of another region or generation.
418    pub fn read_page(
419        &mut self,
420        region: &PageRegion<P>,
421        pos: RegionPos,
422        buf: IoBuf,
423        tag: u64,
424    ) -> Result<(), Rejected> {
425        let state = region.state();
426        let c = ctx(state);
427        if pos.volume != self.store.inner.volume || pos.region != state.id {
428            return Err(rejected(
429                not_written(NotWrittenCause::ForeignPosition, c),
430                buf,
431            ));
432        }
433        if pos.generation != state.generation {
434            return Err(rejected(
435                not_written(NotWrittenCause::StaleGeneration, c),
436                buf,
437            ));
438        }
439        self.read(region.store(), state, pos.offset, buf, tag)
440    }
441
442    fn read(
443        &mut self,
444        store: &Store<P>,
445        state: &Arc<LiveRegion>,
446        offset: u64,
447        mut buf: IoBuf,
448        tag: u64,
449    ) -> Result<(), Rejected> {
450        let pass = admit!(self.admit(store, state, &buf), buf);
451        let len = buf.len() as u64;
452        let block = self.store.block();
453        let c = ctx_range(state, offset, offset.saturating_add(len));
454        if len == 0 {
455            return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
456        }
457        if offset % block != 0 || len % block != 0 {
458            return Err(rejected(not_written(NotWrittenCause::Misaligned, c), buf));
459        }
460        if offset.checked_add(len).is_none_or(|e| e > state.map.len()) {
461            return Err(rejected(not_written(NotWrittenCause::OutOfBounds, c), buf));
462        }
463        let (at, split) = first_piece(state, offset, &mut buf);
464        let req = Req::Read { offset: at, buf };
465        let kind = Kind::Read {
466            start: offset,
467            end: offset + len,
468        };
469        self.issue(req, tag, kind, pass, false, split)
470            .map_err(|(raw, buf)| Rejected {
471                error: match raw {
472                    Some(raw) => Error::Io {
473                        op: Op::Read,
474                        raw,
475                        ctx: c,
476                    },
477                    None => not_written(NotWrittenCause::QueueFull, c),
478                },
479                buf,
480            })
481    }
482
483    /// Submits an admitted operation into a free row. On a refusal the row
484    /// stays free and the pass is released.
485    fn issue(
486        &mut self,
487        req: Req,
488        tag: u64,
489        kind: Kind,
490        pass: OwnedPass<LiveRegion>,
491        durable: bool,
492        split: Option<Split>,
493    ) -> Result<(), (Option<OsError>, Option<IoBuf>)> {
494        // A refusal gives the caller's buffer back whole (a split's first
495        // piece is its front: nothing was moved).
496        let whole = |mut buf: IoBuf| {
497            if let Some(s) = split {
498                let fits = buf.set_len(s.total);
499                debug_assert!(fits, "a split buffer shorter than its operation");
500            }
501            buf
502        };
503        // `admit` checked there is room: a free row always exists here.
504        let Some((index, qtag)) = self.core.table.claim() else {
505            let buf = match req {
506                Req::Write { buf, .. } | Req::Read { buf, .. } => buf,
507            };
508            return Err((None, Some(whole(buf))));
509        };
510        let inner = &*self.store.inner;
511        let file = &inner.file;
512        let (op, write) = match req {
513            Req::Write { offset, buf } => {
514                let bytes = buf.len() as u64;
515                let op = IoOp::Write {
516                    file,
517                    offset,
518                    buf,
519                    dsync: inner.policy.dsync,
520                };
521                (op, Some(bytes))
522            }
523            Req::Read { offset, buf } => (IoOp::Read { file, offset, buf }, None),
524        };
525        // Recorded (and the busy mark published) before the submission: a
526        // queue may do the I/O inside `submit`.
527        self.core.table.start(index, tag, kind, Some(pass), durable);
528        if let Some(s) = split {
529            self.core.table.set_split(index, s);
530        }
531        if let Err(r) = self.core.queue.submit(op, qtag) {
532            self.core.table.abort(index);
533            if r.raw.is_none() {
534                inner.observer.add(Counter::Refused, 1);
535            }
536            return Err((r.raw, r.buf.map(whole)));
537        }
538        if let Some(bytes) = write {
539            inner.observer.add(Counter::Writes, 1);
540            inner.observer.add(Counter::WriteBytes, bytes);
541        }
542        Ok(())
543    }
544
545    /// Hands back every completion that has arrived, without blocking.
546    #[must_use = "completions not taken stay with the submitter and count against its depth"]
547    pub fn reap(&mut self) -> Completions<'_> {
548        let _added = self.core.queue.reap(&mut self.core.raw);
549        self.process();
550        Completions {
551            ready: &mut self.core.table.ready,
552        }
553    }
554
555    /// Blocks until at least `min` completions are ready to hand back (no
556    /// more than are in flight or ready), or the submitter's waker fires,
557    /// then hands back every one that has arrived.
558    ///
559    /// # Errors
560    ///
561    /// `Io` if waiting itself failed: the operations stay in flight and a
562    /// later `reap` or `wait` collects them (a failed wait is never a
563    /// verdict on the media, D-097).
564    #[must_use = "completions not taken stay with the submitter and count against its depth"]
565    pub fn wait(&mut self, min: usize) -> Result<Completions<'_>, Error> {
566        let want = min.min(self.core.table.live + self.core.table.ready.len());
567        if self.core.table.ready.len() >= want {
568            let _added = self.core.queue.reap(&mut self.core.raw);
569            self.process();
570        }
571        while self.core.table.ready.len() < want {
572            // Only the caller's waker ends a wait early: a wake left in the
573            // queue by the flush thread just loops.
574            if self.inbox.take_woken() {
575                break;
576            }
577            let in_queue = self.core.queue.in_flight();
578            if in_queue > 0 {
579                let need = (want - self.core.table.ready.len()).min(in_queue);
580                let waited = if self.inbox.block_in_queue() {
581                    let waited = self.core.queue.wait(need, &mut self.core.raw);
582                    self.inbox.unblock();
583                    waited
584                } else {
585                    // A flush outcome or a wake arrived: no blocking.
586                    Ok(self.core.queue.reap(&mut self.core.raw))
587                };
588                // Whatever arrived is processed, even if the wait then failed.
589                self.process();
590                if let Err(raw) = waited {
591                    return Err(Error::Io {
592                        op: if self.core.table.writes_live() {
593                            Op::Write
594                        } else {
595                            Op::Read
596                        },
597                        raw,
598                        ctx: ErrorContext::default(),
599                    });
600                }
601            } else if self.core.table.asked > 0 {
602                self.inbox.wait(true);
603                self.process();
604            } else {
605                break;
606            }
607        }
608        Ok(Completions {
609            ready: &mut self.core.table.ready,
610        })
611    }
612
613    /// Processes every raw completion collected.
614    fn process(&mut self) {
615        let _unwind = PoisonOnUnwind::new(&self.store);
616        for c in self.core.raw.drain() {
617            // Only a write needing a flush asks (see `Table::finish`).
618            let mut finished = self.core.table.finish(&self.store, c.tag, c.buf, c.result);
619            while let Finish::Next {
620                qtag,
621                offset,
622                buf,
623                write,
624            } = finished
625            {
626                let inner = &*self.store.inner;
627                let file = &inner.file;
628                let bytes = buf.len() as u64;
629                let op = if write {
630                    IoOp::Write {
631                        file,
632                        offset,
633                        buf,
634                        dsync: inner.policy.dsync,
635                    }
636                } else {
637                    IoOp::Read { file, offset, buf }
638                };
639                finished = match self.core.queue.submit(op, qtag) {
640                    Ok(()) => {
641                        if write {
642                            inner.observer.add(Counter::Writes, 1);
643                            inner.observer.add(Counter::WriteBytes, bytes);
644                        }
645                        Finish::Done
646                    }
647                    Err(r) => {
648                        if r.raw.is_none() {
649                            inner.observer.add(Counter::Refused, 1);
650                        }
651                        self.core
652                            .table
653                            .refused_next(&self.store, qtag, r.buf, r.raw)
654                    }
655                };
656            }
657            if let Finish::Flush(qtag, need) = finished {
658                self.core.table.asks.push((qtag, need));
659            }
660        }
661        // Asked once every completion of the pass is finished, the largest
662        // need first, so one flush can cover them all. Taken one at a time:
663        // after a panic the rest stay to be asked for, in the same buffer.
664        while let Some((qtag, need)) = self.core.table.asks.pop() {
665            durable::ask(
666                &mut self.core.table,
667                &self.store,
668                &self.inbox,
669                qtag,
670                need,
671                true,
672            );
673        }
674        // Flush outcomes too, under the same guard.
675        while self.core.table.asked > 0 {
676            let Some((qtag, outcome)) = self.inbox.pop() else {
677                break;
678            };
679            self.core.table.asked -= 1;
680            self.core.table.flushed(&self.store, qtag, outcome);
681        }
682    }
683}
684
685/// Poisons the store if a panic unwinds through the processing of
686/// completions: an operation taken from its row but not settled may be an
687/// append whose reserved range is then never completed.
688struct PoisonOnUnwind<'a, P: Platform>
689where
690    P::Queue: Send,
691{
692    store: &'a Store<P>,
693    /// Already unwinding when created (a drop during another panic): not
694    /// this processing's panic.
695    was: bool,
696}
697
698impl<'a, P: Platform> PoisonOnUnwind<'a, P>
699where
700    P::Queue: Send,
701{
702    fn new(store: &'a Store<P>) -> Self {
703        Self {
704            store,
705            was: std::thread::panicking(),
706        }
707    }
708}
709
710impl<P: Platform> Drop for PoisonOnUnwind<'_, P>
711where
712    P::Queue: Send,
713{
714    fn drop(&mut self) {
715        if !self.was && std::thread::panicking() {
716            self.store.inner.domain.poison(FirstCause {
717                op: Some(Op::Write),
718                raw: None,
719            });
720            self.store.wake_frontiers();
721        }
722    }
723}
724
725impl<P: Platform> Drop for Submitter<P>
726where
727    P::Queue: Send,
728{
729    /// Waits for every operation in flight, untimed (TYP-05), so nothing the
730    /// device may still touch is released. If waiting fails, the queue's own
731    /// drop keeps what the kernel owns, and the store is poisoned when a
732    /// write was in flight: its range may never be written, and nothing of
733    /// its generation may be recycled while it could still land. The same
734    /// if a panic interrupted the processing of a completion (an operation
735    /// left unaccounted may have been an append).
736    fn drop(&mut self) {
737        let mut failed = None;
738        // First every I/O (a durable write's completion may ask for a
739        // flush)...
740        while self.core.table.live > 0 && self.core.queue.in_flight() > 0 {
741            match self
742                .core
743                .queue
744                .wait(self.core.queue.in_flight(), &mut self.core.raw)
745            {
746                Ok(_) => self.process(),
747                Err(raw) => {
748                    failed = Some(raw);
749                    break;
750                }
751            }
752            self.core.table.ready.clear();
753        }
754        // ... with whatever the queue collected before a failed wait...
755        let _added = self.core.queue.reap(&mut self.core.raw);
756        self.process();
757        // ... then every flush asked for: the flush thread answers every
758        // request it accepted.
759        while self.core.table.asked > 0 {
760            self.inbox.wait(false);
761            self.process();
762        }
763        if self.core.table.live > 0
764            && (self.core.table.writes_live() || !self.core.table.consistent())
765        {
766            self.store.inner.domain.poison(FirstCause {
767                op: Some(Op::Write),
768                raw: failed,
769            });
770            self.store.wake_frontiers();
771        }
772        self.store.unregister_submitter(self.id);
773    }
774}