Skip to main content

store_io_engine/submitter/
mod.rs

1//! The engine-layer [`Submitter`] (D-065, D-080, D-098): many operations in
2//! flight from one thread, without blocking, with the same integrity rules
3//! as the blocking calls.
4//!
5//! Every operation is admitted like a blocking call (the region's gate, the
6//! store's state, the same validation), then held in a fixed table until its
7//! completion is processed by [`Submitter::reap`] or [`Submitter::wait`]:
8//!
9//! - its region's **owned pass**, so recycle and release wait for it;
10//! - for an append, its **frontier reservation**, completed only when the
11//!   write transferred every byte (else the store is poisoned: the reserved
12//!   range is a hole no later append may be acknowledged past);
13//! - its tag and submission sequence.
14//!
15//! A [`WriteTicket`](crate::WriteTicket) is minted only from a full
16//! completion (D-080). Queue tags are table indices plus a generation, so a
17//! completion matches exactly one operation. Nothing allocates after
18//! creation. Durable writes and barriers ([`durable`]) wait for a flush
19//! without blocking: the store's flush thread answers them (D-099).
20
21mod durable;
22
23use std::marker::PhantomData;
24use std::sync::Arc;
25use store_io_core::IoClass;
26
27use store_io_core::error::{Error, ErrorContext, FirstCause, NotWrittenCause, Op, OsError};
28use store_io_core::observe::Counter;
29use store_io_platform::{IoBuf, IoOp, Owner, Platform, Queue, QueueConfig};
30
31use crate::completion::{Completions, Rejected, SubmitterConfig, WriteMode};
32use crate::exec::LaneMark;
33use crate::flusher::{Inbox, SubmitterWaker, WakerOf};
34use crate::gate::{Gate, Gated, OwnedPass};
35use crate::io::{ctx, ctx_range, not_written};
36use crate::lifecycle::retired_error;
37use crate::optable::{Finish, Kind, Parked, Split, Table};
38use crate::queue_core::Core;
39use crate::receipt::RegionPos;
40use crate::region::{AppendRegion, PageRegion};
41use crate::store::{LiveRegion, Store};
42
43/// The largest `SubmitterConfig::depth`.
44pub const MAX_DEPTH: u32 = 4096;
45
46impl Gated for LiveRegion {
47    fn gate(&self) -> &Gate {
48        &self.gate
49    }
50}
51
52/// Many reads and writes in flight from one thread (D-065, D-098).
53///
54/// Submitting never blocks: a full submitter is refused with the buffer
55/// given back ([`Rejected`], `NotWritten(QueueFull)`). Completions are
56/// handed back by [`Self::reap`] (never blocks) and [`Self::wait`]
57/// (blocks), with a [`WriteTicket`](crate::WriteTicket) for every write that
58/// completed in full. Buffers come from [`Self::take_buf`] (the store's
59/// pool, shared with its blocking calls: buffers held in flight or kept by
60/// the caller are not available to them) and come back in the completion.
61///
62/// The rules the blocking calls keep hold per operation:
63///
64/// - **An append's position is final at submission** (returned at once),
65///   but it counts as written only once its completion is processed by
66///   this submitter's `reap` or `wait`. Until then, on that region, from
67///   any thread: `sync_through` of any later append waits for it, and an
68///   append starting more than `StoreOptions::append_window` bytes past it
69///   waits (blocking calls) or is refused `NotWritten(TooManyInFlight)`
70///   (submitters). Reap your own appends before syncing or appending far
71///   past them, or the thread waits for itself.
72/// - **Recycle and release wait** for every operation of the submitter on
73///   that region until its completion is processed: a thread must reap its
74///   own submitter's operations on a region before it recycles or releases
75///   that region.
76/// - **A write that fails or transfers short after it was issued** is
77///   `DurabilityUnknown`, and the store is poisoned.
78///
79/// Dropping the submitter waits, untimed, for every operation in flight
80/// before it releases anything the device may still touch. A device that
81/// never completes an operation blocks the drop: the fail-stop choice
82/// (see `Store::oldest_outstanding` for a watchdog).
83///
84/// A submitter belongs to the thread that created it:
85///
86/// ```compile_fail,E0277
87/// fn need_send<T: Send>() {}
88/// need_send::<store_io_engine::Submitter<store_io_sim::SimPlatform>>();
89/// ```
90pub struct Submitter<P: Platform>
91where
92    P::Queue: Send,
93{
94    // Declared first, so its queue drops (after `Drop::drop` drained it)
95    // before the table releases the passes and reservations.
96    core: Core<P::Queue>,
97    /// Where the flush thread answers this submitter's flush requests.
98    inbox: Arc<Inbox<<P::Queue as Queue>::Waker>>,
99    id: u64,
100    pub(crate) store: Store<P>,
101    /// The class of its I/O (`SubmitterConfig::class`).
102    class: IoClass,
103    /// A background submitter's pacing record (one issuer: R5.4).
104    issuer: Option<Arc<crate::qos::Issuer>>,
105    /// The largest piece of one of its operations (`background_io` for a
106    /// background submitter; else unbounded).
107    piece_max: usize,
108    /// Its caller's time away from its calls, for auto-tuning (D-161).
109    away: Away,
110    _thread: PhantomData<*const ()>,
111}
112
113impl<P: Platform> core::fmt::Debug for Submitter<P>
114where
115    P::Queue: Send,
116{
117    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
118        f.debug_struct("Submitter")
119            .field("depth", &self.core.table.depth())
120            .field("in_flight", &self.core.table.live)
121            .field("ready", &self.core.table.ready.len())
122            .finish()
123    }
124}
125
126/// The request [`Submitter::issue`] turns into an [`IoOp`].
127enum Req {
128    Write { offset: u64, buf: IoBuf },
129    Read { offset: u64, buf: IoBuf },
130}
131
132/// The first device operation of `[start, start + buf.len())`: all of it,
133/// or (format 2, across a run's end, spec ยง11) the piece up to the run's
134/// end, `buf` shortened to it, the rest following under the same row
135/// ([`Split`]). Returns its file offset.
136fn first_piece(
137    state: &LiveRegion,
138    start: u64,
139    buf: &mut IoBuf,
140    piece_max: usize,
141) -> (u64, Option<Split>) {
142    let total = buf.len();
143    let piece = (state.map.run_left(start) as usize)
144        .min(total)
145        .min(piece_max);
146    let split = (piece < total).then(|| {
147        let fits = buf.set_len(piece);
148        debug_assert!(fits, "a piece longer than its buffer");
149        Split {
150            total,
151            done: 0,
152            piece_max,
153        }
154    });
155    (state.map.locate(start, piece as u64), split)
156}
157
158fn rejected(error: Error, buf: IoBuf) -> Rejected {
159    Rejected {
160        error,
161        buf: Some(buf),
162    }
163}
164
165/// `?` for admission: refuses with the buffer given back.
166macro_rules! admit {
167    ($e:expr, $buf:ident) => {
168        match $e {
169            Ok(v) => v,
170            Err(error) => return Err(rejected(error, $buf)),
171        }
172    };
173}
174
175impl<P: Platform> Store<P>
176where
177    P::Queue: Send,
178{
179    /// A queue of `depth` operations for one thread alone (a `Submitter`'s,
180    /// a read stream's).
181    pub(crate) fn thread_queue(&self, depth: u32) -> Result<P::Queue, Error> {
182        let qcfg = QueueConfig {
183            depth,
184            parallel: true,
185            pool_threads: self.inner.opts.t3_pool_threads,
186            owner: Owner::Thread,
187        };
188        let inner = &*self.inner;
189        // The file outlives the queue: a registering backend keeps its own
190        // reference to it (`Platform::queue_registered`).
191        let queue = if inner.opts.register_file {
192            inner.platform.queue_registered(qcfg, &inner.file)
193        } else {
194            inner.platform.queue(qcfg)
195        };
196        queue.map_err(|raw| Error::Io {
197            op: Op::Queue,
198            raw,
199            ctx: ErrorContext::default(),
200        })
201    }
202
203    /// A [`Submitter`] for this thread, with its own queue of
204    /// `cfg.depth` operations (1 to [`MAX_DEPTH`]).
205    ///
206    /// # Errors
207    ///
208    /// `Io` if the platform could not create the queue.
209    pub fn submitter(&self, cfg: SubmitterConfig) -> Result<Submitter<P>, Error> {
210        let depth = cfg.depth.clamp(1, MAX_DEPTH);
211        let queue = self.thread_queue(depth)?;
212        let inner = &*self.inner;
213        let n = depth as usize;
214        let mark = Arc::new(LaneMark::new());
215        let inbox = Arc::new(Inbox::new(n, queue.waker()));
216        let id = self.register_submitter(&mark);
217        let background = cfg.class.is_background();
218        Ok(Submitter {
219            core: Core::new(queue, n, mark),
220            inbox,
221            id,
222            store: self.clone(),
223            class: cfg.class,
224            issuer: background.then(|| crate::qos::Issuer::new(&inner.qos)),
225            piece_max: if background {
226                inner.qos.background_io() as usize
227            } else {
228                usize::MAX
229            },
230            away: Away {
231                left: None,
232                total: 0,
233            },
234            _thread: PhantomData,
235        })
236    }
237}
238
239impl<P: Platform> Submitter<P>
240where
241    P::Queue: Send,
242{
243    /// A buffer of `len` bytes from the store's pool, for one operation.
244    ///
245    /// # Errors
246    ///
247    /// `NotWritten(TooLarge)` beyond [`Self::max_buf`];
248    /// `NotWritten(PoolExhausted)` if none is free (for a background
249    /// submitter: if its class's background half is taken).
250    pub fn take_buf(&self, len: usize) -> Result<IoBuf, Error> {
251        if len > self.max_buf() {
252            return Err(not_written(
253                NotWrittenCause::TooLarge,
254                ErrorContext::default(),
255            ));
256        }
257        if self.issuer.is_some() {
258            // Background work never holds more than half of a class (R5.4).
259            return self
260                .store
261                .inner
262                .pool
263                .take_capped(len, usize::MAX)
264                .map_err(|_| not_written(NotWrittenCause::PoolExhausted, ErrorContext::default()));
265        }
266        self.store.take_any(len, ErrorContext::default())
267    }
268
269    /// The largest buffer, and so the largest single operation, in bytes (a
270    /// whole number of blocks): the pool's largest class, `max_write`
271    /// rounded up to a power of two (or `max_io` without large buffers). One
272    /// OS operation (D-101); the kernel may still split it into several
273    /// device requests.
274    #[must_use]
275    pub fn max_buf(&self) -> usize {
276        self.store.max_op()
277    }
278
279    /// Operations submitted whose completions have not been handed to
280    /// `reap` or `wait` yet: I/O in flight, a background submitter's
281    /// operations whose next piece pacing holds back, and durable writes and
282    /// barriers waiting for their flush.
283    #[must_use]
284    pub fn in_flight(&self) -> usize {
285        // Never more than are really pending (an operation lost to a panic
286        // in processing is no longer waited for; the store is poisoned then).
287        // An operation has one piece at a time: in the queue or parked.
288        self.core
289            .table
290            .live
291            .min(self.core.queue.in_flight() + self.core.table.parked.len() + self.core.table.asked)
292    }
293
294    /// Ends this submitter's current or next [`Self::wait`] from another
295    /// thread (D-083), whether it waits for I/O or for a flush.
296    #[must_use]
297    pub fn waker(&self) -> SubmitterWaker<WakerOf<P>> {
298        SubmitterWaker::new(&self.inbox)
299    }
300
301    /// The checks every operation makes first: this store's region, room in
302    /// the table, and the region's gate.
303    fn admit(
304        &self,
305        store: &Store<P>,
306        state: &Arc<LiveRegion>,
307        buf: &IoBuf,
308    ) -> Result<OwnedPass<LiveRegion>, Error> {
309        let c = ctx(state);
310        if !Arc::ptr_eq(&store.inner, &self.store.inner) {
311            return Err(not_written(NotWrittenCause::ForeignPosition, c));
312        }
313        // Only this store's buffers have its alignment and whole blocks of
314        // capacity; any other could fail at the device after an append's
315        // space is reserved.
316        if !self.store.inner.pool.owns(buf) {
317            return Err(not_written(NotWrittenCause::ForeignBuffer, c));
318        }
319        if self.core.table.full() {
320            return Err(not_written(NotWrittenCause::QueueFull, c));
321        }
322        OwnedPass::enter(state).map_err(|why| retired_error(why, state))
323    }
324
325    /// Appends the buffer's bytes (any length of at least one byte) and
326    /// returns their position at once; the completion carries the ticket
327    /// (and, with [`WriteMode::Durable`], the receipt).
328    /// The append starts at a block boundary and the buffer's tail is
329    /// zero-padded in place to the next one (no copy): the buffer comes back
330    /// with its length set to the padded size.
331    ///
332    /// # Errors
333    ///
334    /// [`Rejected`], with the buffer: `NotWritten` (`QueueFull`: the
335    /// submitter is full; `TooManyInFlight`: the region's append ring is
336    /// full, see [`Submitter`]; empty, region full, read-only, not
337    /// positioned, another store's region or buffer), `Poisoned`, or
338    /// `DurabilityUnknown` if the OS refused the write after its position
339    /// was reserved (the store is then poisoned, as for a blocking append).
340    pub fn append(
341        &mut self,
342        region: &AppendRegion<P>,
343        buf: IoBuf,
344        mode: WriteMode,
345        tag: u64,
346    ) -> Result<RegionPos, Rejected> {
347        self.append_with(region, buf, mode, tag, || Ok(()), |()| {})
348    }
349
350    /// [`Self::append`], running `before_reserve` after the append's own
351    /// checks and before its reservation, and handing what it returned to
352    /// `on_reserved` after the reservation and before the write is issued (a
353    /// dependent write's gate passes, `after.rs`). Its error, or a refused
354    /// reservation, is [`Rejected`] with the buffer and nothing reserved.
355    pub(crate) fn append_with<T>(
356        &mut self,
357        region: &AppendRegion<P>,
358        mut buf: IoBuf,
359        mode: WriteMode,
360        tag: u64,
361        before_reserve: impl FnOnce() -> Result<T, Error>,
362        on_reserved: impl FnOnce(T),
363    ) -> Result<RegionPos, Rejected> {
364        let state = region.state();
365        let c = ctx(state);
366        let pass = admit!(self.admit(region.store(), state, &buf), buf);
367        admit!(self.store.ready_for(state), buf);
368        let Some(f) = state.frontier.as_ref() else {
369            return Err(rejected(not_written(NotWrittenCause::NotReady, c), buf));
370        };
371        if !state.positioned.load(std::sync::atomic::Ordering::Acquire) {
372            return Err(rejected(
373                not_written(NotWrittenCause::NotPositioned, c),
374                buf,
375            ));
376        }
377        let len = buf.len();
378        if len == 0 {
379            return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
380        }
381        // Checked before the reservation, so the padding below cannot fail
382        // (the pool's capacities are whole blocks anyway).
383        let block = self.store.block() as usize;
384        if len.div_ceil(block) * block > buf.capacity() {
385            return Err(rejected(not_written(NotWrittenCause::TooLarge, c), buf));
386        }
387        let passes = admit!(before_reserve(), buf);
388        // A background write's first piece is admitted by pacing before any
389        // space is reserved: refused, it is `Paced` with nothing reserved
390        // (R5.4); a refused reservation drops the charge.
391        let charge = admit!(self.admit_first(c), buf);
392        let domain = &self.store.inner.domain;
393        let reserved = f
394            .reserve(len as u64)
395            .map_err(|why| match domain.poisoned() {
396                Some(first) => Error::Poisoned { first },
397                None => not_written(why, c),
398            });
399        let r = admit!(reserved, buf);
400        // Held until the row holds it: a panic before then (in `locate`, or
401        // a backend's `submit`) poisons and wakes every waiter at once,
402        // never a hole that a later barrier waits on until this submitter
403        // drops. The store is the region's (`admit` checked it is this one).
404        let held = region.store().hold_reservation(r);
405        let padded = (r.end - r.start) as usize;
406        buf.as_mut_capacity()[len..padded].fill(0);
407        let _fits = buf.set_len(padded);
408        let pos = RegionPos {
409            volume: self.store.inner.volume,
410            region: state.id,
411            generation: state.generation,
412            offset: r.start,
413        };
414        let (offset, split) = first_piece(state, r.start, &mut buf, self.piece_max);
415        let req = Req::Write { offset, buf };
416        let durable = mode == WriteMode::Durable;
417        // Never held through the issue: on a queue that performs at once the
418        // write runs inside it (R5.3).
419        on_reserved(passes);
420        let issued = self.issue(req, tag, Kind::Append(r), pass, durable, split, charge);
421        let _r = held.disarm();
422        match issued {
423            Ok(()) => Ok(pos),
424            // Past the reservation: a hole, as for a blocking append.
425            Err((raw, buf)) => Err(Rejected {
426                error: self
427                    .store
428                    .unknown(Op::Write, raw, ctx_range(state, r.start, r.data_end)),
429                buf,
430            }),
431        }
432    }
433
434    /// Writes the buffer (whole blocks) at `pos`; the completion carries the
435    /// ticket (and, with [`WriteMode::Durable`], the receipt).
436    ///
437    /// # Errors
438    ///
439    /// [`Rejected`], with the buffer: `NotWritten` (foreign, stale,
440    /// misaligned, out of range, submitter full, read-only, another store's
441    /// buffer), `Poisoned`, or `Io` if the OS refused it (nothing reached
442    /// the device).
443    pub fn write(
444        &mut self,
445        region: &PageRegion<P>,
446        pos: RegionPos,
447        buf: IoBuf,
448        mode: WriteMode,
449        tag: u64,
450    ) -> Result<(), Rejected> {
451        self.write_with(region, pos, buf, mode, tag, || Ok(()))
452    }
453
454    /// [`Self::write`], running `before_issue` after the write's own checks
455    /// and before its buffer is cut to the first run (so a refusal gives
456    /// back the whole buffer); what it returns is dropped before the write
457    /// is issued. Its error is [`Rejected`] with the buffer.
458    pub(crate) fn write_with<T>(
459        &mut self,
460        region: &PageRegion<P>,
461        pos: RegionPos,
462        mut buf: IoBuf,
463        mode: WriteMode,
464        tag: u64,
465        before_issue: impl FnOnce() -> Result<T, Error>,
466    ) -> Result<(), Rejected> {
467        let state = region.state();
468        let pass = admit!(self.admit(region.store(), state, &buf), buf);
469        admit!(self.store.ready_for(state), buf);
470        let end = admit!(region.check(&pos, buf.len()), buf);
471        let c = ctx_range(state, pos.offset, end);
472        let passes = admit!(before_issue(), buf);
473        // A background write is admitted by pacing before it is issued.
474        let charge = admit!(self.admit_first(c), buf);
475        let (offset, split) = first_piece(state, pos.offset, &mut buf, self.piece_max);
476        let req = Req::Write { offset, buf };
477        let kind = Kind::Page {
478            start: pos.offset,
479            end,
480        };
481        drop(passes);
482        self.issue(
483            req,
484            tag,
485            kind,
486            pass,
487            mode == WriteMode::Durable,
488            split,
489            charge,
490        )
491        .map_err(|(raw, buf)| Rejected {
492            error: Store::<P>::refused(raw, c),
493            buf,
494        })
495    }
496
497    /// Reads into the buffer (whole blocks) from the block-aligned `offset`
498    /// of an append region; the completion carries the bytes. Reads work on
499    /// read-only and poisoned stores. A background submitter's reads are
500    /// paced as its writes are.
501    ///
502    /// # Errors
503    ///
504    /// [`Rejected`], with the buffer: `NotWritten` (empty, misaligned, out
505    /// of range, submitter full, another store's region or buffer; `Paced`
506    /// for a background submitter while commits are active: reap and
507    /// retry), or `Io` if the OS refused it.
508    pub fn read_append(
509        &mut self,
510        region: &AppendRegion<P>,
511        offset: u64,
512        buf: IoBuf,
513        tag: u64,
514    ) -> Result<(), Rejected> {
515        self.read(region.store(), region.state(), offset, buf, tag)
516    }
517
518    /// Reads into the buffer (whole blocks) at `pos` of a page region; the
519    /// completion carries the bytes.
520    ///
521    /// # Errors
522    ///
523    /// As [`Self::read_append`], and `NotWritten(ForeignPosition |
524    /// StaleGeneration)` for a position of another region or generation.
525    pub fn read_page(
526        &mut self,
527        region: &PageRegion<P>,
528        pos: RegionPos,
529        buf: IoBuf,
530        tag: u64,
531    ) -> Result<(), Rejected> {
532        let state = region.state();
533        let c = ctx(state);
534        if pos.volume != self.store.inner.volume || pos.region != state.id {
535            return Err(rejected(
536                not_written(NotWrittenCause::ForeignPosition, c),
537                buf,
538            ));
539        }
540        if pos.generation != state.generation {
541            return Err(rejected(
542                not_written(NotWrittenCause::StaleGeneration, c),
543                buf,
544            ));
545        }
546        self.read(region.store(), state, pos.offset, buf, tag)
547    }
548
549    fn read(
550        &mut self,
551        store: &Store<P>,
552        state: &Arc<LiveRegion>,
553        offset: u64,
554        mut buf: IoBuf,
555        tag: u64,
556    ) -> Result<(), Rejected> {
557        let pass = admit!(self.admit(store, state, &buf), buf);
558        let len = buf.len() as u64;
559        let block = self.store.block();
560        let c = ctx_range(state, offset, offset.saturating_add(len));
561        if len == 0 {
562            return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
563        }
564        if offset % block != 0 || len % block != 0 {
565            return Err(rejected(not_written(NotWrittenCause::Misaligned, c), buf));
566        }
567        if offset.checked_add(len).is_none_or(|e| e > state.map.len()) {
568            return Err(rejected(not_written(NotWrittenCause::OutOfBounds, c), buf));
569        }
570        // A background read is admitted by pacing as a write is (R5.4: the
571        // class is the submitter's).
572        let charge = admit!(self.admit_first(c), buf);
573        let (at, split) = first_piece(state, offset, &mut buf, self.piece_max);
574        let req = Req::Read { offset: at, buf };
575        let kind = Kind::Read {
576            start: offset,
577            end: offset + len,
578        };
579        self.issue(req, tag, kind, pass, false, split, charge)
580            .map_err(|(raw, buf)| Rejected {
581                error: match raw {
582                    Some(raw) => Error::Io {
583                        op: Op::Read,
584                        raw,
585                        ctx: c,
586                    },
587                    None => not_written(NotWrittenCause::QueueFull, c),
588                },
589                buf,
590            })
591    }
592
593    /// Submits an admitted operation into a free row. On a refusal the row
594    /// stays free and the pass is released.
595    // An operation's request, tag, kind, pass, durability, split and pacing
596    // charge are all needed, and none derives from another.
597    #[allow(clippy::too_many_arguments)]
598    fn issue(
599        &mut self,
600        req: Req,
601        tag: u64,
602        kind: Kind,
603        pass: OwnedPass<LiveRegion>,
604        durable: bool,
605        split: Option<Split>,
606        charge: Option<crate::qos::Charge>,
607    ) -> Result<(), (Option<OsError>, Option<IoBuf>)> {
608        // A refusal gives the caller's buffer back whole (a split's first
609        // piece is its front: nothing was moved).
610        let whole = |mut buf: IoBuf| {
611            if let Some(s) = split {
612                let fits = buf.set_len(s.total);
613                debug_assert!(fits, "a split buffer shorter than its operation");
614            }
615            buf
616        };
617        // `admit` checked there is room: a free row always exists here.
618        let Some((index, qtag)) = self.core.table.claim() else {
619            let buf = match req {
620                Req::Write { buf, .. } | Req::Read { buf, .. } => buf,
621            };
622            return Err((None, Some(whole(buf))));
623        };
624        let inner = &*self.store.inner;
625        let file = &inner.file;
626        let (op, write) = match req {
627            Req::Write { offset, buf } => {
628                let bytes = buf.len() as u64;
629                // Marks commits active; its completion is collected later
630                // and elsewhere, so auto-tuning takes no sample of it
631                // (D-158: a device of async power-safe commits alone stays
632                // at the floor).
633                if inner.policy.dsync && !self.class.is_background() {
634                    let _entry = inner.domain.note_commit();
635                }
636                let op = IoOp::Write {
637                    file,
638                    offset,
639                    buf,
640                    dsync: inner.policy.dsync,
641                };
642                (op, Some(bytes))
643            }
644            Req::Read { offset, buf } => (IoOp::Read { file, offset, buf }, None),
645        };
646        // A durable commit's flush is timed for auto-tuning from before its
647        // write, as a blocking one's (D-160), read before the submission,
648        // with its caller's time away kept beside it (D-161). `None` where
649        // the write is the commit (`write_entry`: a `dsync` store's region
650        // not forced); a durable write that needs no flush asks for none,
651        // and its entry goes unused. A background submitter's flushes are
652        // never samples.
653        self.away.enter(&inner.domain);
654        let entry = if durable && !self.class.is_background() {
655            self.store
656                .write_entry(pass.get())
657                .map(|at| (at, self.away.total))
658        } else {
659            None
660        };
661        // Recorded (and the busy mark published) before the submission: a
662        // queue may do the I/O inside `submit`.
663        self.core.table.start(index, tag, kind, Some(pass), durable);
664        self.core.table.set_entry(index, entry);
665        if let Some(s) = split {
666            self.core.table.set_split(index, s);
667        }
668        // Held by the row: released when the piece completes, or with the
669        // row (a refused submission below aborts it).
670        self.core.table.set_charge(qtag, charge);
671        let submitted = self.core.queue.submit(op, qtag);
672        self.away.leave(&inner.domain, self.core.table.entries);
673        if let Err(r) = submitted {
674            self.core.table.abort(index);
675            if r.raw.is_none() {
676                inner.observer.add(Counter::Refused, 1);
677            }
678            return Err((r.raw, r.buf.map(whole)));
679        }
680        if let Some(bytes) = write {
681            inner.observer.add(Counter::Writes, 1);
682            inner.observer.add(Counter::WriteBytes, bytes);
683        }
684        Ok(())
685    }
686
687    /// Hands back every completion that has arrived, without blocking.
688    #[must_use = "completions not taken stay with the submitter and count against its depth"]
689    pub fn reap(&mut self) -> Completions<'_> {
690        self.away.enter(&self.store.inner.domain);
691        let _added = self.core.queue.reap(&mut self.core.raw);
692        self.process();
693        self.away
694            .leave(&self.store.inner.domain, self.core.table.entries);
695        Completions {
696            ready: &mut self.core.table.ready,
697        }
698    }
699
700    /// Blocks until at least `min` completions are ready to hand back (no
701    /// more than are in flight or ready), or the submitter's waker fires,
702    /// then hands back every one that has arrived.
703    ///
704    /// # Errors
705    ///
706    /// `Io` if waiting itself failed: the operations stay in flight and a
707    /// later `reap` or `wait` collects them (a failed wait is never a
708    /// verdict on the media, D-097).
709    #[must_use = "completions not taken stay with the submitter and count against its depth"]
710    pub fn wait(&mut self, min: usize) -> Result<Completions<'_>, Error> {
711        self.away.enter(&self.store.inner.domain);
712        let want = min.min(self.core.table.live + self.core.table.ready.len());
713        if self.core.table.ready.len() >= want {
714            let _added = self.core.queue.reap(&mut self.core.raw);
715            self.process();
716        }
717        while self.core.table.ready.len() < want {
718            // Only the caller's waker ends a wait early: a wake left in the
719            // queue by the flush thread just loops.
720            if self.inbox.take_woken() {
721                break;
722            }
723            let in_queue = self.core.queue.in_flight();
724            if in_queue > 0 {
725                let need = (want - self.core.table.ready.len()).min(in_queue);
726                let waited = if self.inbox.block_in_queue() {
727                    let waited = self.core.queue.wait(need, &mut self.core.raw);
728                    self.inbox.unblock();
729                    waited
730                } else {
731                    // A flush outcome or a wake arrived: no blocking.
732                    Ok(self.core.queue.reap(&mut self.core.raw))
733                };
734                // Whatever arrived is processed, even if the wait then failed.
735                self.process();
736                if let Err(raw) = waited {
737                    self.away
738                        .leave(&self.store.inner.domain, self.core.table.entries);
739                    return Err(Error::Io {
740                        op: if self.core.table.writes_live() {
741                            Op::Write
742                        } else {
743                            Op::Read
744                        },
745                        raw,
746                        ctx: ErrorContext::default(),
747                    });
748                }
749            } else if self.core.table.asked > 0 {
750                self.inbox.wait(true);
751                self.process();
752            } else if !self.core.table.parked.is_empty() {
753                // Only parked pieces: nothing of this submitter in flight, so
754                // the oldest goes out (R5.4: a wait never ends early on pieces
755                // pacing held back).
756                self.unpark_idle();
757            } else {
758                break;
759            }
760        }
761        self.away
762            .leave(&self.store.inner.domain, self.core.table.entries);
763        Ok(Completions {
764            ready: &mut self.core.table.ready,
765        })
766    }
767
768    /// Processes every raw completion collected, then resubmits parked
769    /// pieces while pacing admits them, then asks for the flushes the pass
770    /// wants and takes the answers.
771    fn process(&mut self) {
772        self.process_with(true);
773    }
774
775    /// [`Self::process`]; with `issue` false nothing more is submitted, no
776    /// parked piece and no next piece (parked instead: a drop whose wait
777    /// failed, fail-stop; its live writes poison the store).
778    fn process_with(&mut self, issue: bool) {
779        let class = self.barrier_class();
780        let Self {
781            core,
782            inbox,
783            store,
784            issuer,
785            away,
786            ..
787        } = self;
788        let _unwind = PoisonOnUnwind::new(store);
789        let mut pieces = Pieces {
790            table: &mut core.table,
791            queue: &mut core.queue,
792            store,
793            issuer: issuer.as_ref(),
794            issue,
795            away: away.total,
796        };
797        for c in core.raw.drain() {
798            // Only a write needing a flush asks (see `Table::finish`).
799            let finished = pieces.table.finish(store, c.tag, c.buf, c.result);
800            pieces.follow(finished);
801        }
802        if issue {
803            pieces.unpark(false);
804        }
805        pieces.ask_and_take(inbox, class);
806    }
807
808    /// Resubmits parked pieces with nothing in flight (a wait, a drop): the
809    /// oldest always goes out.
810    fn unpark_idle(&mut self) {
811        let class = self.barrier_class();
812        let Self {
813            core,
814            inbox,
815            store,
816            issuer,
817            away,
818            ..
819        } = self;
820        let _unwind = PoisonOnUnwind::new(store);
821        let mut pieces = Pieces {
822            table: &mut core.table,
823            queue: &mut core.queue,
824            store,
825            issuer: issuer.as_ref(),
826            issue: true,
827            away: away.total,
828        };
829        pieces.unpark(true);
830        pieces.ask_and_take(inbox, class);
831    }
832
833    /// The class its barriers run as: a background submitter's never mark
834    /// commits active; a read-class submitter's are commit-class (its writes
835    /// are).
836    pub(crate) fn barrier_class(&self) -> IoClass {
837        if self.class.is_background() {
838            IoClass::Background
839        } else {
840            IoClass::Commit
841        }
842    }
843
844    /// Admits a background operation's first piece (R5.4): `None` for a
845    /// submitter that is not background.
846    fn admit_first(&self, c: ErrorContext) -> Result<Option<crate::qos::Charge>, Error> {
847        match &self.issuer {
848            None => Ok(None),
849            Some(i) => match i.admit(self.store.inner.platform.monotonic_ns()) {
850                Some(charge) => Ok(Some(charge)),
851                None => Err(not_written(NotWrittenCause::Paced, c)),
852            },
853        }
854    }
855}
856
857/// A submitter's caller's time away from its calls, for auto-tuning
858/// (D-161): a durable write's flush is asked for only when a later `reap`,
859/// `wait` or the drop processes its completion, so its sample, timed from
860/// before the write, leaves out the time the caller spent elsewhere
861/// meanwhile (its own work between calls is no commit's latency). Its
862/// time in the calls (a blocking wait, other submissions) stays in. Kept
863/// only while a row holds an entry (`Table::entries`): no clock read, and
864/// no check of the group's tuning, otherwise (a background submitter's
865/// rows never hold one).
866struct Away {
867    /// When the caller last left a call, while an entry waits (`None`
868    /// otherwise, or with auto-tuning off).
869    left: Option<u64>,
870    /// The time away in all.
871    total: u64,
872}
873
874impl Away {
875    /// The caller is back in a call: the time since it last left one is
876    /// added up.
877    fn enter(&mut self, domain: &crate::domain::Domain) {
878        if let Some(left) = self.left.take() {
879            if let Some(now) = domain.arrival() {
880                self.total = self.total.saturating_add(now.saturating_sub(left));
881            }
882        }
883    }
884
885    /// The caller leaves a call: when, if `entries` rows hold an entry
886    /// (then a check of the group's tuning and one clock read).
887    fn leave(&mut self, domain: &crate::domain::Domain, entries: usize) {
888        self.left = if entries > 0 { domain.arrival() } else { None };
889    }
890}
891
892/// What processing a pass's pieces needs of a submitter, borrowed field by
893/// field (so the raw completions can be drained beside it).
894struct Pieces<'a, P: Platform>
895where
896    P::Queue: Send,
897{
898    table: &'a mut Table,
899    queue: &'a mut P::Queue,
900    store: &'a Store<P>,
901    issuer: Option<&'a Arc<crate::qos::Issuer>>,
902    /// Pieces may be submitted (false: every next piece is parked).
903    issue: bool,
904    /// The submitter's caller's time away in all (`Away::total`).
905    away: u64,
906}
907
908impl<P: Platform> Pieces<'_, P>
909where
910    P::Queue: Send,
911{
912    /// Follows a completion through every next piece it leads to; a flush
913    /// it wants is asked for at the end of the pass.
914    fn follow(&mut self, mut finished: Finish) {
915        while let Finish::Next {
916            qtag,
917            offset,
918            buf,
919            write,
920        } = finished
921        {
922            finished = self.next_piece(qtag, offset, buf, write);
923        }
924        if let Finish::Flush(qtag, need, entry) = finished {
925            // Later by the time its caller spent away since its entry.
926            let entry = entry.map(|(at, away)| at.saturating_add(self.away.saturating_sub(away)));
927            self.table.asks.push((qtag, need, entry));
928        }
929    }
930
931    /// Pacing's admission of one more piece: `Some(None)` for a submitter
932    /// that is not background, `None` when refused.
933    fn admit(&self) -> Option<Option<crate::qos::Charge>> {
934        match self.issuer {
935            None => Some(None),
936            Some(i) => i.admit(self.store.inner.platform.monotonic_ns()).map(Some),
937        }
938    }
939
940    /// The next piece of an operation across pieces: never a write piece to
941    /// a poisoned store (fail-stop: the operation ends `Poisoned`); a
942    /// background piece admitted by pacing, or parked, never dropped; behind
943    /// any piece parked before it (oldest first: the pass's `unpark`
944    /// submits them in order); parked, never submitted, once a drop's wait
945    /// has failed.
946    fn next_piece(&mut self, qtag: u64, offset: u64, buf: IoBuf, write: bool) -> Finish {
947        if write {
948            if let Some(first) = self.store.inner.domain.poisoned() {
949                return self
950                    .table
951                    .refused_poisoned(self.store, qtag, Some(buf), first);
952            }
953        }
954        let admitted = if self.issue && self.table.parked.is_empty() {
955            self.admit()
956        } else {
957            None
958        };
959        let Some(charge) = admitted else {
960            self.table.parked.push_back(Parked {
961                qtag,
962                offset,
963                buf,
964                write,
965            });
966            return Finish::Done;
967        };
968        self.submit_piece(qtag, offset, buf, write, charge)
969    }
970
971    /// Submits a piece, its charge held by its row.
972    fn submit_piece(
973        &mut self,
974        qtag: u64,
975        offset: u64,
976        buf: IoBuf,
977        write: bool,
978        charge: Option<crate::qos::Charge>,
979    ) -> Finish {
980        let inner = &*self.store.inner;
981        let file = &inner.file;
982        let bytes = buf.len() as u64;
983        self.table.set_charge(qtag, charge);
984        let op = if write {
985            IoOp::Write {
986                file,
987                offset,
988                buf,
989                dsync: inner.policy.dsync,
990            }
991        } else {
992            IoOp::Read { file, offset, buf }
993        };
994        match self.queue.submit(op, qtag) {
995            Ok(()) => {
996                if write {
997                    inner.observer.add(Counter::Writes, 1);
998                    inner.observer.add(Counter::WriteBytes, bytes);
999                }
1000                Finish::Done
1001            }
1002            Err(r) => {
1003                if r.raw.is_none() {
1004                    inner.observer.add(Counter::Refused, 1);
1005                }
1006                self.table.refused_next(self.store, qtag, r.buf, r.raw)
1007            }
1008        }
1009    }
1010
1011    /// Resubmits parked pieces, oldest first, while pacing admits them. With
1012    /// `force`, the oldest is submitted (uncharged) even if refused when
1013    /// nothing at all is in flight: pacing admits a call with nothing in
1014    /// flight anyway, so this changes nothing in a correct run, and a drain
1015    /// or a wait with only parked pieces can never spin.
1016    fn unpark(&mut self, force: bool) {
1017        while let Some(p) = self.table.parked.pop_front() {
1018            let Parked {
1019                qtag,
1020                offset,
1021                buf,
1022                write,
1023            } = p;
1024            if write {
1025                if let Some(first) = self.store.inner.domain.poisoned() {
1026                    let _done = self
1027                        .table
1028                        .refused_poisoned(self.store, qtag, Some(buf), first);
1029                    continue;
1030                }
1031            }
1032            let charge = match self.admit() {
1033                Some(charge) => charge,
1034                None if force && self.queue.in_flight() == 0 => {
1035                    // Pacing admits an issuer with nothing in flight, and a
1036                    // piece's charge goes at its completion: so only a
1037                    // charge that outlived its piece is refused here.
1038                    // (After a panic in processing, a completion dropped
1039                    // unprocessed keeps its row's charge: the store is
1040                    // poisoned then, and no write goes out.)
1041                    debug_assert!(
1042                        self.issuer.is_none_or(|i| i.in_flight() == 0)
1043                            || self.store.inner.domain.poisoned().is_some(),
1044                        "a charge outlived its piece"
1045                    );
1046                    None
1047                }
1048                None => {
1049                    self.table.parked.push_front(Parked {
1050                        qtag,
1051                        offset,
1052                        buf,
1053                        write,
1054                    });
1055                    return;
1056                }
1057            };
1058            let finished = self.submit_piece(qtag, offset, buf, write, charge);
1059            self.follow(finished);
1060        }
1061    }
1062
1063    /// Asks the flush thread for what the pass wants, and takes its answers.
1064    fn ask_and_take(&mut self, inbox: &Arc<Inbox<<P::Queue as Queue>::Waker>>, class: IoClass) {
1065        // Asked once every completion of the pass is finished, the largest
1066        // need first, so one flush can cover them all. Taken one at a time:
1067        // after a panic the rest stay to be asked for, in the same buffer.
1068        while let Some((qtag, need, entry)) = self.table.asks.pop() {
1069            durable::ask(
1070                self.table, self.store, inbox, qtag, need, true, class, entry,
1071            );
1072        }
1073        // Flush outcomes too, under the same guard.
1074        while self.table.asked > 0 {
1075            let Some((qtag, outcome)) = inbox.pop() else {
1076                break;
1077            };
1078            self.table.asked -= 1;
1079            self.table.flushed(self.store, qtag, outcome);
1080        }
1081    }
1082}
1083
1084/// Poisons the store if a panic unwinds through the processing of
1085/// completions: an operation taken from its row but not settled may be an
1086/// append whose reserved range is then never completed.
1087struct PoisonOnUnwind<'a, P: Platform>
1088where
1089    P::Queue: Send,
1090{
1091    store: &'a Store<P>,
1092    /// Already unwinding when created (a drop during another panic): not
1093    /// this processing's panic.
1094    was: bool,
1095}
1096
1097impl<'a, P: Platform> PoisonOnUnwind<'a, P>
1098where
1099    P::Queue: Send,
1100{
1101    fn new(store: &'a Store<P>) -> Self {
1102        Self {
1103            store,
1104            was: std::thread::panicking(),
1105        }
1106    }
1107}
1108
1109impl<P: Platform> Drop for PoisonOnUnwind<'_, P>
1110where
1111    P::Queue: Send,
1112{
1113    fn drop(&mut self) {
1114        if !self.was && std::thread::panicking() {
1115            self.store.inner.domain.poison(FirstCause {
1116                op: Some(Op::Write),
1117                raw: None,
1118            });
1119            self.store.wake_frontiers();
1120        }
1121    }
1122}
1123
1124impl<P: Platform> Drop for Submitter<P>
1125where
1126    P::Queue: Send,
1127{
1128    /// Waits for every operation in flight, untimed (TYP-05), so nothing the
1129    /// device may still touch is released. If waiting fails, the queue's own
1130    /// drop keeps what the kernel owns, nothing more is submitted (no
1131    /// parked piece, no next piece), and the store is poisoned when a write
1132    /// was in flight or unfinished: its range may never be written, and
1133    /// nothing of its generation may be recycled while it could still land.
1134    /// The same if a panic interrupted the processing of a completion (an
1135    /// operation left unaccounted may have been an append).
1136    fn drop(&mut self) {
1137        self.away.enter(&self.store.inner.domain);
1138        let mut failed = None;
1139        // First every I/O (a durable write's completion may ask for a
1140        // flush), parked pieces submitted as pacing admits them (a parked
1141        // piece is a live write: never dropped while the waits succeed; with
1142        // nothing in flight the oldest is always admitted)...
1143        while self.core.table.live > 0
1144            && (self.core.queue.in_flight() > 0 || !self.core.table.parked.is_empty())
1145        {
1146            if self.core.queue.in_flight() == 0 {
1147                self.unpark_idle();
1148                continue;
1149            }
1150            match self
1151                .core
1152                .queue
1153                .wait(self.core.queue.in_flight(), &mut self.core.raw)
1154            {
1155                Ok(_) => self.process(),
1156                Err(raw) => {
1157                    failed = Some(raw);
1158                    break;
1159                }
1160            }
1161            self.core.table.ready.clear();
1162        }
1163        // ... with whatever the queue collected before a failed wait (and
1164        // then nothing more submitted: the store is poisoned below)...
1165        let issue = failed.is_none();
1166        let _added = self.core.queue.reap(&mut self.core.raw);
1167        self.process_with(issue);
1168        // ... then every flush asked for: the flush thread answers every
1169        // request it accepted.
1170        while self.core.table.asked > 0 {
1171            self.inbox.wait(false);
1172            self.process_with(issue);
1173        }
1174        if self.core.table.live > 0
1175            && (self.core.table.writes_live() || !self.core.table.consistent())
1176        {
1177            self.store.inner.domain.poison(FirstCause {
1178                op: Some(Op::Write),
1179                raw: failed,
1180            });
1181            self.store.wake_frontiers();
1182        }
1183        self.store.unregister_submitter(self.id);
1184    }
1185}