store_io_engine/submitter/mod.rs
1//! The engine-layer [`Submitter`] (D-065, D-080, D-098): many operations in
2//! flight from one thread, without blocking, with the same integrity rules
3//! as the blocking calls.
4//!
5//! Every operation is admitted like a blocking call (the region's gate, the
6//! store's state, the same validation), then held in a fixed table until its
7//! completion is processed by [`Submitter::reap`] or [`Submitter::wait`]:
8//!
9//! - its region's **owned pass**, so recycle and release wait for it;
10//! - for an append, its **frontier reservation**, completed only when the
11//! write transferred every byte (else the store is poisoned: the reserved
12//! range is a hole no later append may be acknowledged past);
13//! - its tag and submission sequence.
14//!
15//! A [`WriteTicket`](crate::WriteTicket) is minted only from a full
16//! completion (D-080). Queue tags are table indices plus a generation, so a
17//! completion matches exactly one operation. Nothing allocates after
18//! creation. Durable writes and barriers ([`durable`]) wait for a flush
19//! without blocking: the store's flush thread answers them (D-099).
20
21mod durable;
22
23use std::marker::PhantomData;
24use std::sync::Arc;
25use store_io_core::IoClass;
26
27use store_io_core::error::{Error, ErrorContext, FirstCause, NotWrittenCause, Op, OsError};
28use store_io_core::observe::Counter;
29use store_io_platform::{IoBuf, IoOp, Owner, Platform, Queue, QueueConfig};
30
31use crate::completion::{Completions, Rejected, SubmitterConfig, WriteMode};
32use crate::exec::LaneMark;
33use crate::flusher::{Inbox, SubmitterWaker, WakerOf};
34use crate::gate::{Gate, Gated, OwnedPass};
35use crate::io::{ctx, ctx_range, not_written};
36use crate::lifecycle::retired_error;
37use crate::optable::{Finish, Kind, Parked, Split, Table};
38use crate::queue_core::Core;
39use crate::receipt::RegionPos;
40use crate::region::{AppendRegion, PageRegion};
41use crate::store::{LiveRegion, Store};
42
43/// The largest `SubmitterConfig::depth`.
44pub const MAX_DEPTH: u32 = 4096;
45
46impl Gated for LiveRegion {
47 fn gate(&self) -> &Gate {
48 &self.gate
49 }
50}
51
52/// Many reads and writes in flight from one thread (D-065, D-098).
53///
54/// Submitting never blocks: a full submitter is refused with the buffer
55/// given back ([`Rejected`], `NotWritten(QueueFull)`). Completions are
56/// handed back by [`Self::reap`] (never blocks) and [`Self::wait`]
57/// (blocks), with a [`WriteTicket`](crate::WriteTicket) for every write that
58/// completed in full. Buffers come from [`Self::take_buf`] (the store's
59/// pool, shared with its blocking calls: buffers held in flight or kept by
60/// the caller are not available to them) and come back in the completion.
61///
62/// The rules the blocking calls keep hold per operation:
63///
64/// - **An append's position is final at submission** (returned at once),
65/// but it counts as written only once its completion is processed by
66/// this submitter's `reap` or `wait`. Until then, on that region, from
67/// any thread: `sync_through` of any later append waits for it, and an
68/// append starting more than `StoreOptions::append_window` bytes past it
69/// waits (blocking calls) or is refused `NotWritten(TooManyInFlight)`
70/// (submitters). Reap your own appends before syncing or appending far
71/// past them, or the thread waits for itself.
72/// - **Recycle and release wait** for every operation of the submitter on
73/// that region until its completion is processed: a thread must reap its
74/// own submitter's operations on a region before it recycles or releases
75/// that region.
76/// - **A write that fails or transfers short after it was issued** is
77/// `DurabilityUnknown`, and the store is poisoned.
78///
79/// Dropping the submitter waits, untimed, for every operation in flight
80/// before it releases anything the device may still touch. A device that
81/// never completes an operation blocks the drop: the fail-stop choice
82/// (see `Store::oldest_outstanding` for a watchdog).
83///
84/// A submitter belongs to the thread that created it:
85///
86/// ```compile_fail,E0277
87/// fn need_send<T: Send>() {}
88/// need_send::<store_io_engine::Submitter<store_io_sim::SimPlatform>>();
89/// ```
90pub struct Submitter<P: Platform>
91where
92 P::Queue: Send,
93{
94 // Declared first, so its queue drops (after `Drop::drop` drained it)
95 // before the table releases the passes and reservations.
96 core: Core<P::Queue>,
97 /// Where the flush thread answers this submitter's flush requests.
98 inbox: Arc<Inbox<<P::Queue as Queue>::Waker>>,
99 id: u64,
100 pub(crate) store: Store<P>,
101 /// The class of its I/O (`SubmitterConfig::class`).
102 class: IoClass,
103 /// A background submitter's pacing record (one issuer: R5.4).
104 issuer: Option<Arc<crate::qos::Issuer>>,
105 /// The largest piece of one of its operations (`background_io` for a
106 /// background submitter; else unbounded).
107 piece_max: usize,
108 /// Its caller's time away from its calls, for auto-tuning (D-161).
109 away: Away,
110 _thread: PhantomData<*const ()>,
111}
112
113impl<P: Platform> core::fmt::Debug for Submitter<P>
114where
115 P::Queue: Send,
116{
117 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
118 f.debug_struct("Submitter")
119 .field("depth", &self.core.table.depth())
120 .field("in_flight", &self.core.table.live)
121 .field("ready", &self.core.table.ready.len())
122 .finish()
123 }
124}
125
126/// The request [`Submitter::issue`] turns into an [`IoOp`].
127enum Req {
128 Write { offset: u64, buf: IoBuf },
129 Read { offset: u64, buf: IoBuf },
130}
131
132/// The first device operation of `[start, start + buf.len())`: all of it,
133/// or (format 2, across a run's end, spec ยง11) the piece up to the run's
134/// end, `buf` shortened to it, the rest following under the same row
135/// ([`Split`]). Returns its file offset.
136fn first_piece(
137 state: &LiveRegion,
138 start: u64,
139 buf: &mut IoBuf,
140 piece_max: usize,
141) -> (u64, Option<Split>) {
142 let total = buf.len();
143 let piece = (state.map.run_left(start) as usize)
144 .min(total)
145 .min(piece_max);
146 let split = (piece < total).then(|| {
147 let fits = buf.set_len(piece);
148 debug_assert!(fits, "a piece longer than its buffer");
149 Split {
150 total,
151 done: 0,
152 piece_max,
153 }
154 });
155 (state.map.locate(start, piece as u64), split)
156}
157
158fn rejected(error: Error, buf: IoBuf) -> Rejected {
159 Rejected {
160 error,
161 buf: Some(buf),
162 }
163}
164
165/// `?` for admission: refuses with the buffer given back.
166macro_rules! admit {
167 ($e:expr, $buf:ident) => {
168 match $e {
169 Ok(v) => v,
170 Err(error) => return Err(rejected(error, $buf)),
171 }
172 };
173}
174
175impl<P: Platform> Store<P>
176where
177 P::Queue: Send,
178{
179 /// A queue of `depth` operations for one thread alone (a `Submitter`'s,
180 /// a read stream's).
181 pub(crate) fn thread_queue(&self, depth: u32) -> Result<P::Queue, Error> {
182 let qcfg = QueueConfig {
183 depth,
184 parallel: true,
185 pool_threads: self.inner.opts.t3_pool_threads,
186 owner: Owner::Thread,
187 };
188 let inner = &*self.inner;
189 // The file outlives the queue: a registering backend keeps its own
190 // reference to it (`Platform::queue_registered`).
191 let queue = if inner.opts.register_file {
192 inner.platform.queue_registered(qcfg, &inner.file)
193 } else {
194 inner.platform.queue(qcfg)
195 };
196 queue.map_err(|raw| Error::Io {
197 op: Op::Queue,
198 raw,
199 ctx: ErrorContext::default(),
200 })
201 }
202
203 /// A [`Submitter`] for this thread, with its own queue of
204 /// `cfg.depth` operations (1 to [`MAX_DEPTH`]).
205 ///
206 /// # Errors
207 ///
208 /// `Io` if the platform could not create the queue.
209 pub fn submitter(&self, cfg: SubmitterConfig) -> Result<Submitter<P>, Error> {
210 let depth = cfg.depth.clamp(1, MAX_DEPTH);
211 let queue = self.thread_queue(depth)?;
212 let inner = &*self.inner;
213 let n = depth as usize;
214 let mark = Arc::new(LaneMark::new());
215 let inbox = Arc::new(Inbox::new(n, queue.waker()));
216 let id = self.register_submitter(&mark);
217 let background = cfg.class.is_background();
218 Ok(Submitter {
219 core: Core::new(queue, n, mark),
220 inbox,
221 id,
222 store: self.clone(),
223 class: cfg.class,
224 issuer: background.then(|| crate::qos::Issuer::new(&inner.qos)),
225 piece_max: if background {
226 inner.qos.background_io() as usize
227 } else {
228 usize::MAX
229 },
230 away: Away {
231 left: None,
232 total: 0,
233 },
234 _thread: PhantomData,
235 })
236 }
237}
238
239impl<P: Platform> Submitter<P>
240where
241 P::Queue: Send,
242{
243 /// A buffer of `len` bytes from the store's pool, for one operation.
244 ///
245 /// # Errors
246 ///
247 /// `NotWritten(TooLarge)` beyond [`Self::max_buf`];
248 /// `NotWritten(PoolExhausted)` if none is free (for a background
249 /// submitter: if its class's background half is taken).
250 pub fn take_buf(&self, len: usize) -> Result<IoBuf, Error> {
251 if len > self.max_buf() {
252 return Err(not_written(
253 NotWrittenCause::TooLarge,
254 ErrorContext::default(),
255 ));
256 }
257 if self.issuer.is_some() {
258 // Background work never holds more than half of a class (R5.4).
259 return self
260 .store
261 .inner
262 .pool
263 .take_capped(len, usize::MAX)
264 .map_err(|_| not_written(NotWrittenCause::PoolExhausted, ErrorContext::default()));
265 }
266 self.store.take_any(len, ErrorContext::default())
267 }
268
269 /// The largest buffer, and so the largest single operation, in bytes (a
270 /// whole number of blocks): the pool's largest class, `max_write`
271 /// rounded up to a power of two (or `max_io` without large buffers). One
272 /// OS operation (D-101); the kernel may still split it into several
273 /// device requests.
274 #[must_use]
275 pub fn max_buf(&self) -> usize {
276 self.store.max_op()
277 }
278
279 /// Operations submitted whose completions have not been handed to
280 /// `reap` or `wait` yet: I/O in flight, a background submitter's
281 /// operations whose next piece pacing holds back, and durable writes and
282 /// barriers waiting for their flush.
283 #[must_use]
284 pub fn in_flight(&self) -> usize {
285 // Never more than are really pending (an operation lost to a panic
286 // in processing is no longer waited for; the store is poisoned then).
287 // An operation has one piece at a time: in the queue or parked.
288 self.core
289 .table
290 .live
291 .min(self.core.queue.in_flight() + self.core.table.parked.len() + self.core.table.asked)
292 }
293
294 /// Ends this submitter's current or next [`Self::wait`] from another
295 /// thread (D-083), whether it waits for I/O or for a flush.
296 #[must_use]
297 pub fn waker(&self) -> SubmitterWaker<WakerOf<P>> {
298 SubmitterWaker::new(&self.inbox)
299 }
300
301 /// The checks every operation makes first: this store's region, room in
302 /// the table, and the region's gate.
303 fn admit(
304 &self,
305 store: &Store<P>,
306 state: &Arc<LiveRegion>,
307 buf: &IoBuf,
308 ) -> Result<OwnedPass<LiveRegion>, Error> {
309 let c = ctx(state);
310 if !Arc::ptr_eq(&store.inner, &self.store.inner) {
311 return Err(not_written(NotWrittenCause::ForeignPosition, c));
312 }
313 // Only this store's buffers have its alignment and whole blocks of
314 // capacity; any other could fail at the device after an append's
315 // space is reserved.
316 if !self.store.inner.pool.owns(buf) {
317 return Err(not_written(NotWrittenCause::ForeignBuffer, c));
318 }
319 if self.core.table.full() {
320 return Err(not_written(NotWrittenCause::QueueFull, c));
321 }
322 OwnedPass::enter(state).map_err(|why| retired_error(why, state))
323 }
324
325 /// Appends the buffer's bytes (any length of at least one byte) and
326 /// returns their position at once; the completion carries the ticket
327 /// (and, with [`WriteMode::Durable`], the receipt).
328 /// The append starts at a block boundary and the buffer's tail is
329 /// zero-padded in place to the next one (no copy): the buffer comes back
330 /// with its length set to the padded size.
331 ///
332 /// # Errors
333 ///
334 /// [`Rejected`], with the buffer: `NotWritten` (`QueueFull`: the
335 /// submitter is full; `TooManyInFlight`: the region's append ring is
336 /// full, see [`Submitter`]; empty, region full, read-only, not
337 /// positioned, another store's region or buffer), `Poisoned`, or
338 /// `DurabilityUnknown` if the OS refused the write after its position
339 /// was reserved (the store is then poisoned, as for a blocking append).
340 pub fn append(
341 &mut self,
342 region: &AppendRegion<P>,
343 buf: IoBuf,
344 mode: WriteMode,
345 tag: u64,
346 ) -> Result<RegionPos, Rejected> {
347 self.append_with(region, buf, mode, tag, || Ok(()), |()| {})
348 }
349
350 /// [`Self::append`], running `before_reserve` after the append's own
351 /// checks and before its reservation, and handing what it returned to
352 /// `on_reserved` after the reservation and before the write is issued (a
353 /// dependent write's gate passes, `after.rs`). Its error, or a refused
354 /// reservation, is [`Rejected`] with the buffer and nothing reserved.
355 pub(crate) fn append_with<T>(
356 &mut self,
357 region: &AppendRegion<P>,
358 mut buf: IoBuf,
359 mode: WriteMode,
360 tag: u64,
361 before_reserve: impl FnOnce() -> Result<T, Error>,
362 on_reserved: impl FnOnce(T),
363 ) -> Result<RegionPos, Rejected> {
364 let state = region.state();
365 let c = ctx(state);
366 let pass = admit!(self.admit(region.store(), state, &buf), buf);
367 admit!(self.store.ready_for(state), buf);
368 let Some(f) = state.frontier.as_ref() else {
369 return Err(rejected(not_written(NotWrittenCause::NotReady, c), buf));
370 };
371 if !state.positioned.load(std::sync::atomic::Ordering::Acquire) {
372 return Err(rejected(
373 not_written(NotWrittenCause::NotPositioned, c),
374 buf,
375 ));
376 }
377 let len = buf.len();
378 if len == 0 {
379 return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
380 }
381 // Checked before the reservation, so the padding below cannot fail
382 // (the pool's capacities are whole blocks anyway).
383 let block = self.store.block() as usize;
384 if len.div_ceil(block) * block > buf.capacity() {
385 return Err(rejected(not_written(NotWrittenCause::TooLarge, c), buf));
386 }
387 let passes = admit!(before_reserve(), buf);
388 // A background write's first piece is admitted by pacing before any
389 // space is reserved: refused, it is `Paced` with nothing reserved
390 // (R5.4); a refused reservation drops the charge.
391 let charge = admit!(self.admit_first(c), buf);
392 let domain = &self.store.inner.domain;
393 let reserved = f
394 .reserve(len as u64)
395 .map_err(|why| match domain.poisoned() {
396 Some(first) => Error::Poisoned { first },
397 None => not_written(why, c),
398 });
399 let r = admit!(reserved, buf);
400 // Held until the row holds it: a panic before then (in `locate`, or
401 // a backend's `submit`) poisons and wakes every waiter at once,
402 // never a hole that a later barrier waits on until this submitter
403 // drops. The store is the region's (`admit` checked it is this one).
404 let held = region.store().hold_reservation(r);
405 let padded = (r.end - r.start) as usize;
406 buf.as_mut_capacity()[len..padded].fill(0);
407 let _fits = buf.set_len(padded);
408 let pos = RegionPos {
409 volume: self.store.inner.volume,
410 region: state.id,
411 generation: state.generation,
412 offset: r.start,
413 };
414 let (offset, split) = first_piece(state, r.start, &mut buf, self.piece_max);
415 let req = Req::Write { offset, buf };
416 let durable = mode == WriteMode::Durable;
417 // Never held through the issue: on a queue that performs at once the
418 // write runs inside it (R5.3).
419 on_reserved(passes);
420 let issued = self.issue(req, tag, Kind::Append(r), pass, durable, split, charge);
421 let _r = held.disarm();
422 match issued {
423 Ok(()) => Ok(pos),
424 // Past the reservation: a hole, as for a blocking append.
425 Err((raw, buf)) => Err(Rejected {
426 error: self
427 .store
428 .unknown(Op::Write, raw, ctx_range(state, r.start, r.data_end)),
429 buf,
430 }),
431 }
432 }
433
434 /// Writes the buffer (whole blocks) at `pos`; the completion carries the
435 /// ticket (and, with [`WriteMode::Durable`], the receipt).
436 ///
437 /// # Errors
438 ///
439 /// [`Rejected`], with the buffer: `NotWritten` (foreign, stale,
440 /// misaligned, out of range, submitter full, read-only, another store's
441 /// buffer), `Poisoned`, or `Io` if the OS refused it (nothing reached
442 /// the device).
443 pub fn write(
444 &mut self,
445 region: &PageRegion<P>,
446 pos: RegionPos,
447 buf: IoBuf,
448 mode: WriteMode,
449 tag: u64,
450 ) -> Result<(), Rejected> {
451 self.write_with(region, pos, buf, mode, tag, || Ok(()))
452 }
453
454 /// [`Self::write`], running `before_issue` after the write's own checks
455 /// and before its buffer is cut to the first run (so a refusal gives
456 /// back the whole buffer); what it returns is dropped before the write
457 /// is issued. Its error is [`Rejected`] with the buffer.
458 pub(crate) fn write_with<T>(
459 &mut self,
460 region: &PageRegion<P>,
461 pos: RegionPos,
462 mut buf: IoBuf,
463 mode: WriteMode,
464 tag: u64,
465 before_issue: impl FnOnce() -> Result<T, Error>,
466 ) -> Result<(), Rejected> {
467 let state = region.state();
468 let pass = admit!(self.admit(region.store(), state, &buf), buf);
469 admit!(self.store.ready_for(state), buf);
470 let end = admit!(region.check(&pos, buf.len()), buf);
471 let c = ctx_range(state, pos.offset, end);
472 let passes = admit!(before_issue(), buf);
473 // A background write is admitted by pacing before it is issued.
474 let charge = admit!(self.admit_first(c), buf);
475 let (offset, split) = first_piece(state, pos.offset, &mut buf, self.piece_max);
476 let req = Req::Write { offset, buf };
477 let kind = Kind::Page {
478 start: pos.offset,
479 end,
480 };
481 drop(passes);
482 self.issue(
483 req,
484 tag,
485 kind,
486 pass,
487 mode == WriteMode::Durable,
488 split,
489 charge,
490 )
491 .map_err(|(raw, buf)| Rejected {
492 error: Store::<P>::refused(raw, c),
493 buf,
494 })
495 }
496
497 /// Reads into the buffer (whole blocks) from the block-aligned `offset`
498 /// of an append region; the completion carries the bytes. Reads work on
499 /// read-only and poisoned stores. A background submitter's reads are
500 /// paced as its writes are.
501 ///
502 /// # Errors
503 ///
504 /// [`Rejected`], with the buffer: `NotWritten` (empty, misaligned, out
505 /// of range, submitter full, another store's region or buffer; `Paced`
506 /// for a background submitter while commits are active: reap and
507 /// retry), or `Io` if the OS refused it.
508 pub fn read_append(
509 &mut self,
510 region: &AppendRegion<P>,
511 offset: u64,
512 buf: IoBuf,
513 tag: u64,
514 ) -> Result<(), Rejected> {
515 self.read(region.store(), region.state(), offset, buf, tag)
516 }
517
518 /// Reads into the buffer (whole blocks) at `pos` of a page region; the
519 /// completion carries the bytes.
520 ///
521 /// # Errors
522 ///
523 /// As [`Self::read_append`], and `NotWritten(ForeignPosition |
524 /// StaleGeneration)` for a position of another region or generation.
525 pub fn read_page(
526 &mut self,
527 region: &PageRegion<P>,
528 pos: RegionPos,
529 buf: IoBuf,
530 tag: u64,
531 ) -> Result<(), Rejected> {
532 let state = region.state();
533 let c = ctx(state);
534 if pos.volume != self.store.inner.volume || pos.region != state.id {
535 return Err(rejected(
536 not_written(NotWrittenCause::ForeignPosition, c),
537 buf,
538 ));
539 }
540 if pos.generation != state.generation {
541 return Err(rejected(
542 not_written(NotWrittenCause::StaleGeneration, c),
543 buf,
544 ));
545 }
546 self.read(region.store(), state, pos.offset, buf, tag)
547 }
548
549 fn read(
550 &mut self,
551 store: &Store<P>,
552 state: &Arc<LiveRegion>,
553 offset: u64,
554 mut buf: IoBuf,
555 tag: u64,
556 ) -> Result<(), Rejected> {
557 let pass = admit!(self.admit(store, state, &buf), buf);
558 let len = buf.len() as u64;
559 let block = self.store.block();
560 let c = ctx_range(state, offset, offset.saturating_add(len));
561 if len == 0 {
562 return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
563 }
564 if offset % block != 0 || len % block != 0 {
565 return Err(rejected(not_written(NotWrittenCause::Misaligned, c), buf));
566 }
567 if offset.checked_add(len).is_none_or(|e| e > state.map.len()) {
568 return Err(rejected(not_written(NotWrittenCause::OutOfBounds, c), buf));
569 }
570 // A background read is admitted by pacing as a write is (R5.4: the
571 // class is the submitter's).
572 let charge = admit!(self.admit_first(c), buf);
573 let (at, split) = first_piece(state, offset, &mut buf, self.piece_max);
574 let req = Req::Read { offset: at, buf };
575 let kind = Kind::Read {
576 start: offset,
577 end: offset + len,
578 };
579 self.issue(req, tag, kind, pass, false, split, charge)
580 .map_err(|(raw, buf)| Rejected {
581 error: match raw {
582 Some(raw) => Error::Io {
583 op: Op::Read,
584 raw,
585 ctx: c,
586 },
587 None => not_written(NotWrittenCause::QueueFull, c),
588 },
589 buf,
590 })
591 }
592
593 /// Submits an admitted operation into a free row. On a refusal the row
594 /// stays free and the pass is released.
595 // An operation's request, tag, kind, pass, durability, split and pacing
596 // charge are all needed, and none derives from another.
597 #[allow(clippy::too_many_arguments)]
598 fn issue(
599 &mut self,
600 req: Req,
601 tag: u64,
602 kind: Kind,
603 pass: OwnedPass<LiveRegion>,
604 durable: bool,
605 split: Option<Split>,
606 charge: Option<crate::qos::Charge>,
607 ) -> Result<(), (Option<OsError>, Option<IoBuf>)> {
608 // A refusal gives the caller's buffer back whole (a split's first
609 // piece is its front: nothing was moved).
610 let whole = |mut buf: IoBuf| {
611 if let Some(s) = split {
612 let fits = buf.set_len(s.total);
613 debug_assert!(fits, "a split buffer shorter than its operation");
614 }
615 buf
616 };
617 // `admit` checked there is room: a free row always exists here.
618 let Some((index, qtag)) = self.core.table.claim() else {
619 let buf = match req {
620 Req::Write { buf, .. } | Req::Read { buf, .. } => buf,
621 };
622 return Err((None, Some(whole(buf))));
623 };
624 let inner = &*self.store.inner;
625 let file = &inner.file;
626 let (op, write) = match req {
627 Req::Write { offset, buf } => {
628 let bytes = buf.len() as u64;
629 // Marks commits active; its completion is collected later
630 // and elsewhere, so auto-tuning takes no sample of it
631 // (D-158: a device of async power-safe commits alone stays
632 // at the floor).
633 if inner.policy.dsync && !self.class.is_background() {
634 let _entry = inner.domain.note_commit();
635 }
636 let op = IoOp::Write {
637 file,
638 offset,
639 buf,
640 dsync: inner.policy.dsync,
641 };
642 (op, Some(bytes))
643 }
644 Req::Read { offset, buf } => (IoOp::Read { file, offset, buf }, None),
645 };
646 // A durable commit's flush is timed for auto-tuning from before its
647 // write, as a blocking one's (D-160), read before the submission,
648 // with its caller's time away kept beside it (D-161). `None` where
649 // the write is the commit (`write_entry`: a `dsync` store's region
650 // not forced); a durable write that needs no flush asks for none,
651 // and its entry goes unused. A background submitter's flushes are
652 // never samples.
653 self.away.enter(&inner.domain);
654 let entry = if durable && !self.class.is_background() {
655 self.store
656 .write_entry(pass.get())
657 .map(|at| (at, self.away.total))
658 } else {
659 None
660 };
661 // Recorded (and the busy mark published) before the submission: a
662 // queue may do the I/O inside `submit`.
663 self.core.table.start(index, tag, kind, Some(pass), durable);
664 self.core.table.set_entry(index, entry);
665 if let Some(s) = split {
666 self.core.table.set_split(index, s);
667 }
668 // Held by the row: released when the piece completes, or with the
669 // row (a refused submission below aborts it).
670 self.core.table.set_charge(qtag, charge);
671 let submitted = self.core.queue.submit(op, qtag);
672 self.away.leave(&inner.domain, self.core.table.entries);
673 if let Err(r) = submitted {
674 self.core.table.abort(index);
675 if r.raw.is_none() {
676 inner.observer.add(Counter::Refused, 1);
677 }
678 return Err((r.raw, r.buf.map(whole)));
679 }
680 if let Some(bytes) = write {
681 inner.observer.add(Counter::Writes, 1);
682 inner.observer.add(Counter::WriteBytes, bytes);
683 }
684 Ok(())
685 }
686
687 /// Hands back every completion that has arrived, without blocking.
688 #[must_use = "completions not taken stay with the submitter and count against its depth"]
689 pub fn reap(&mut self) -> Completions<'_> {
690 self.away.enter(&self.store.inner.domain);
691 let _added = self.core.queue.reap(&mut self.core.raw);
692 self.process();
693 self.away
694 .leave(&self.store.inner.domain, self.core.table.entries);
695 Completions {
696 ready: &mut self.core.table.ready,
697 }
698 }
699
700 /// Blocks until at least `min` completions are ready to hand back (no
701 /// more than are in flight or ready), or the submitter's waker fires,
702 /// then hands back every one that has arrived.
703 ///
704 /// # Errors
705 ///
706 /// `Io` if waiting itself failed: the operations stay in flight and a
707 /// later `reap` or `wait` collects them (a failed wait is never a
708 /// verdict on the media, D-097).
709 #[must_use = "completions not taken stay with the submitter and count against its depth"]
710 pub fn wait(&mut self, min: usize) -> Result<Completions<'_>, Error> {
711 self.away.enter(&self.store.inner.domain);
712 let want = min.min(self.core.table.live + self.core.table.ready.len());
713 if self.core.table.ready.len() >= want {
714 let _added = self.core.queue.reap(&mut self.core.raw);
715 self.process();
716 }
717 while self.core.table.ready.len() < want {
718 // Only the caller's waker ends a wait early: a wake left in the
719 // queue by the flush thread just loops.
720 if self.inbox.take_woken() {
721 break;
722 }
723 let in_queue = self.core.queue.in_flight();
724 if in_queue > 0 {
725 let need = (want - self.core.table.ready.len()).min(in_queue);
726 let waited = if self.inbox.block_in_queue() {
727 let waited = self.core.queue.wait(need, &mut self.core.raw);
728 self.inbox.unblock();
729 waited
730 } else {
731 // A flush outcome or a wake arrived: no blocking.
732 Ok(self.core.queue.reap(&mut self.core.raw))
733 };
734 // Whatever arrived is processed, even if the wait then failed.
735 self.process();
736 if let Err(raw) = waited {
737 self.away
738 .leave(&self.store.inner.domain, self.core.table.entries);
739 return Err(Error::Io {
740 op: if self.core.table.writes_live() {
741 Op::Write
742 } else {
743 Op::Read
744 },
745 raw,
746 ctx: ErrorContext::default(),
747 });
748 }
749 } else if self.core.table.asked > 0 {
750 self.inbox.wait(true);
751 self.process();
752 } else if !self.core.table.parked.is_empty() {
753 // Only parked pieces: nothing of this submitter in flight, so
754 // the oldest goes out (R5.4: a wait never ends early on pieces
755 // pacing held back).
756 self.unpark_idle();
757 } else {
758 break;
759 }
760 }
761 self.away
762 .leave(&self.store.inner.domain, self.core.table.entries);
763 Ok(Completions {
764 ready: &mut self.core.table.ready,
765 })
766 }
767
768 /// Processes every raw completion collected, then resubmits parked
769 /// pieces while pacing admits them, then asks for the flushes the pass
770 /// wants and takes the answers.
771 fn process(&mut self) {
772 self.process_with(true);
773 }
774
775 /// [`Self::process`]; with `issue` false nothing more is submitted, no
776 /// parked piece and no next piece (parked instead: a drop whose wait
777 /// failed, fail-stop; its live writes poison the store).
778 fn process_with(&mut self, issue: bool) {
779 let class = self.barrier_class();
780 let Self {
781 core,
782 inbox,
783 store,
784 issuer,
785 away,
786 ..
787 } = self;
788 let _unwind = PoisonOnUnwind::new(store);
789 let mut pieces = Pieces {
790 table: &mut core.table,
791 queue: &mut core.queue,
792 store,
793 issuer: issuer.as_ref(),
794 issue,
795 away: away.total,
796 };
797 for c in core.raw.drain() {
798 // Only a write needing a flush asks (see `Table::finish`).
799 let finished = pieces.table.finish(store, c.tag, c.buf, c.result);
800 pieces.follow(finished);
801 }
802 if issue {
803 pieces.unpark(false);
804 }
805 pieces.ask_and_take(inbox, class);
806 }
807
808 /// Resubmits parked pieces with nothing in flight (a wait, a drop): the
809 /// oldest always goes out.
810 fn unpark_idle(&mut self) {
811 let class = self.barrier_class();
812 let Self {
813 core,
814 inbox,
815 store,
816 issuer,
817 away,
818 ..
819 } = self;
820 let _unwind = PoisonOnUnwind::new(store);
821 let mut pieces = Pieces {
822 table: &mut core.table,
823 queue: &mut core.queue,
824 store,
825 issuer: issuer.as_ref(),
826 issue: true,
827 away: away.total,
828 };
829 pieces.unpark(true);
830 pieces.ask_and_take(inbox, class);
831 }
832
833 /// The class its barriers run as: a background submitter's never mark
834 /// commits active; a read-class submitter's are commit-class (its writes
835 /// are).
836 pub(crate) fn barrier_class(&self) -> IoClass {
837 if self.class.is_background() {
838 IoClass::Background
839 } else {
840 IoClass::Commit
841 }
842 }
843
844 /// Admits a background operation's first piece (R5.4): `None` for a
845 /// submitter that is not background.
846 fn admit_first(&self, c: ErrorContext) -> Result<Option<crate::qos::Charge>, Error> {
847 match &self.issuer {
848 None => Ok(None),
849 Some(i) => match i.admit(self.store.inner.platform.monotonic_ns()) {
850 Some(charge) => Ok(Some(charge)),
851 None => Err(not_written(NotWrittenCause::Paced, c)),
852 },
853 }
854 }
855}
856
857/// A submitter's caller's time away from its calls, for auto-tuning
858/// (D-161): a durable write's flush is asked for only when a later `reap`,
859/// `wait` or the drop processes its completion, so its sample, timed from
860/// before the write, leaves out the time the caller spent elsewhere
861/// meanwhile (its own work between calls is no commit's latency). Its
862/// time in the calls (a blocking wait, other submissions) stays in. Kept
863/// only while a row holds an entry (`Table::entries`): no clock read, and
864/// no check of the group's tuning, otherwise (a background submitter's
865/// rows never hold one).
866struct Away {
867 /// When the caller last left a call, while an entry waits (`None`
868 /// otherwise, or with auto-tuning off).
869 left: Option<u64>,
870 /// The time away in all.
871 total: u64,
872}
873
874impl Away {
875 /// The caller is back in a call: the time since it last left one is
876 /// added up.
877 fn enter(&mut self, domain: &crate::domain::Domain) {
878 if let Some(left) = self.left.take() {
879 if let Some(now) = domain.arrival() {
880 self.total = self.total.saturating_add(now.saturating_sub(left));
881 }
882 }
883 }
884
885 /// The caller leaves a call: when, if `entries` rows hold an entry
886 /// (then a check of the group's tuning and one clock read).
887 fn leave(&mut self, domain: &crate::domain::Domain, entries: usize) {
888 self.left = if entries > 0 { domain.arrival() } else { None };
889 }
890}
891
892/// What processing a pass's pieces needs of a submitter, borrowed field by
893/// field (so the raw completions can be drained beside it).
894struct Pieces<'a, P: Platform>
895where
896 P::Queue: Send,
897{
898 table: &'a mut Table,
899 queue: &'a mut P::Queue,
900 store: &'a Store<P>,
901 issuer: Option<&'a Arc<crate::qos::Issuer>>,
902 /// Pieces may be submitted (false: every next piece is parked).
903 issue: bool,
904 /// The submitter's caller's time away in all (`Away::total`).
905 away: u64,
906}
907
908impl<P: Platform> Pieces<'_, P>
909where
910 P::Queue: Send,
911{
912 /// Follows a completion through every next piece it leads to; a flush
913 /// it wants is asked for at the end of the pass.
914 fn follow(&mut self, mut finished: Finish) {
915 while let Finish::Next {
916 qtag,
917 offset,
918 buf,
919 write,
920 } = finished
921 {
922 finished = self.next_piece(qtag, offset, buf, write);
923 }
924 if let Finish::Flush(qtag, need, entry) = finished {
925 // Later by the time its caller spent away since its entry.
926 let entry = entry.map(|(at, away)| at.saturating_add(self.away.saturating_sub(away)));
927 self.table.asks.push((qtag, need, entry));
928 }
929 }
930
931 /// Pacing's admission of one more piece: `Some(None)` for a submitter
932 /// that is not background, `None` when refused.
933 fn admit(&self) -> Option<Option<crate::qos::Charge>> {
934 match self.issuer {
935 None => Some(None),
936 Some(i) => i.admit(self.store.inner.platform.monotonic_ns()).map(Some),
937 }
938 }
939
940 /// The next piece of an operation across pieces: never a write piece to
941 /// a poisoned store (fail-stop: the operation ends `Poisoned`); a
942 /// background piece admitted by pacing, or parked, never dropped; behind
943 /// any piece parked before it (oldest first: the pass's `unpark`
944 /// submits them in order); parked, never submitted, once a drop's wait
945 /// has failed.
946 fn next_piece(&mut self, qtag: u64, offset: u64, buf: IoBuf, write: bool) -> Finish {
947 if write {
948 if let Some(first) = self.store.inner.domain.poisoned() {
949 return self
950 .table
951 .refused_poisoned(self.store, qtag, Some(buf), first);
952 }
953 }
954 let admitted = if self.issue && self.table.parked.is_empty() {
955 self.admit()
956 } else {
957 None
958 };
959 let Some(charge) = admitted else {
960 self.table.parked.push_back(Parked {
961 qtag,
962 offset,
963 buf,
964 write,
965 });
966 return Finish::Done;
967 };
968 self.submit_piece(qtag, offset, buf, write, charge)
969 }
970
971 /// Submits a piece, its charge held by its row.
972 fn submit_piece(
973 &mut self,
974 qtag: u64,
975 offset: u64,
976 buf: IoBuf,
977 write: bool,
978 charge: Option<crate::qos::Charge>,
979 ) -> Finish {
980 let inner = &*self.store.inner;
981 let file = &inner.file;
982 let bytes = buf.len() as u64;
983 self.table.set_charge(qtag, charge);
984 let op = if write {
985 IoOp::Write {
986 file,
987 offset,
988 buf,
989 dsync: inner.policy.dsync,
990 }
991 } else {
992 IoOp::Read { file, offset, buf }
993 };
994 match self.queue.submit(op, qtag) {
995 Ok(()) => {
996 if write {
997 inner.observer.add(Counter::Writes, 1);
998 inner.observer.add(Counter::WriteBytes, bytes);
999 }
1000 Finish::Done
1001 }
1002 Err(r) => {
1003 if r.raw.is_none() {
1004 inner.observer.add(Counter::Refused, 1);
1005 }
1006 self.table.refused_next(self.store, qtag, r.buf, r.raw)
1007 }
1008 }
1009 }
1010
1011 /// Resubmits parked pieces, oldest first, while pacing admits them. With
1012 /// `force`, the oldest is submitted (uncharged) even if refused when
1013 /// nothing at all is in flight: pacing admits a call with nothing in
1014 /// flight anyway, so this changes nothing in a correct run, and a drain
1015 /// or a wait with only parked pieces can never spin.
1016 fn unpark(&mut self, force: bool) {
1017 while let Some(p) = self.table.parked.pop_front() {
1018 let Parked {
1019 qtag,
1020 offset,
1021 buf,
1022 write,
1023 } = p;
1024 if write {
1025 if let Some(first) = self.store.inner.domain.poisoned() {
1026 let _done = self
1027 .table
1028 .refused_poisoned(self.store, qtag, Some(buf), first);
1029 continue;
1030 }
1031 }
1032 let charge = match self.admit() {
1033 Some(charge) => charge,
1034 None if force && self.queue.in_flight() == 0 => {
1035 // Pacing admits an issuer with nothing in flight, and a
1036 // piece's charge goes at its completion: so only a
1037 // charge that outlived its piece is refused here.
1038 // (After a panic in processing, a completion dropped
1039 // unprocessed keeps its row's charge: the store is
1040 // poisoned then, and no write goes out.)
1041 debug_assert!(
1042 self.issuer.is_none_or(|i| i.in_flight() == 0)
1043 || self.store.inner.domain.poisoned().is_some(),
1044 "a charge outlived its piece"
1045 );
1046 None
1047 }
1048 None => {
1049 self.table.parked.push_front(Parked {
1050 qtag,
1051 offset,
1052 buf,
1053 write,
1054 });
1055 return;
1056 }
1057 };
1058 let finished = self.submit_piece(qtag, offset, buf, write, charge);
1059 self.follow(finished);
1060 }
1061 }
1062
1063 /// Asks the flush thread for what the pass wants, and takes its answers.
1064 fn ask_and_take(&mut self, inbox: &Arc<Inbox<<P::Queue as Queue>::Waker>>, class: IoClass) {
1065 // Asked once every completion of the pass is finished, the largest
1066 // need first, so one flush can cover them all. Taken one at a time:
1067 // after a panic the rest stay to be asked for, in the same buffer.
1068 while let Some((qtag, need, entry)) = self.table.asks.pop() {
1069 durable::ask(
1070 self.table, self.store, inbox, qtag, need, true, class, entry,
1071 );
1072 }
1073 // Flush outcomes too, under the same guard.
1074 while self.table.asked > 0 {
1075 let Some((qtag, outcome)) = inbox.pop() else {
1076 break;
1077 };
1078 self.table.asked -= 1;
1079 self.table.flushed(self.store, qtag, outcome);
1080 }
1081 }
1082}
1083
1084/// Poisons the store if a panic unwinds through the processing of
1085/// completions: an operation taken from its row but not settled may be an
1086/// append whose reserved range is then never completed.
1087struct PoisonOnUnwind<'a, P: Platform>
1088where
1089 P::Queue: Send,
1090{
1091 store: &'a Store<P>,
1092 /// Already unwinding when created (a drop during another panic): not
1093 /// this processing's panic.
1094 was: bool,
1095}
1096
1097impl<'a, P: Platform> PoisonOnUnwind<'a, P>
1098where
1099 P::Queue: Send,
1100{
1101 fn new(store: &'a Store<P>) -> Self {
1102 Self {
1103 store,
1104 was: std::thread::panicking(),
1105 }
1106 }
1107}
1108
1109impl<P: Platform> Drop for PoisonOnUnwind<'_, P>
1110where
1111 P::Queue: Send,
1112{
1113 fn drop(&mut self) {
1114 if !self.was && std::thread::panicking() {
1115 self.store.inner.domain.poison(FirstCause {
1116 op: Some(Op::Write),
1117 raw: None,
1118 });
1119 self.store.wake_frontiers();
1120 }
1121 }
1122}
1123
1124impl<P: Platform> Drop for Submitter<P>
1125where
1126 P::Queue: Send,
1127{
1128 /// Waits for every operation in flight, untimed (TYP-05), so nothing the
1129 /// device may still touch is released. If waiting fails, the queue's own
1130 /// drop keeps what the kernel owns, nothing more is submitted (no
1131 /// parked piece, no next piece), and the store is poisoned when a write
1132 /// was in flight or unfinished: its range may never be written, and
1133 /// nothing of its generation may be recycled while it could still land.
1134 /// The same if a panic interrupted the processing of a completion (an
1135 /// operation left unaccounted may have been an append).
1136 fn drop(&mut self) {
1137 self.away.enter(&self.store.inner.domain);
1138 let mut failed = None;
1139 // First every I/O (a durable write's completion may ask for a
1140 // flush), parked pieces submitted as pacing admits them (a parked
1141 // piece is a live write: never dropped while the waits succeed; with
1142 // nothing in flight the oldest is always admitted)...
1143 while self.core.table.live > 0
1144 && (self.core.queue.in_flight() > 0 || !self.core.table.parked.is_empty())
1145 {
1146 if self.core.queue.in_flight() == 0 {
1147 self.unpark_idle();
1148 continue;
1149 }
1150 match self
1151 .core
1152 .queue
1153 .wait(self.core.queue.in_flight(), &mut self.core.raw)
1154 {
1155 Ok(_) => self.process(),
1156 Err(raw) => {
1157 failed = Some(raw);
1158 break;
1159 }
1160 }
1161 self.core.table.ready.clear();
1162 }
1163 // ... with whatever the queue collected before a failed wait (and
1164 // then nothing more submitted: the store is poisoned below)...
1165 let issue = failed.is_none();
1166 let _added = self.core.queue.reap(&mut self.core.raw);
1167 self.process_with(issue);
1168 // ... then every flush asked for: the flush thread answers every
1169 // request it accepted.
1170 while self.core.table.asked > 0 {
1171 self.inbox.wait(false);
1172 self.process_with(issue);
1173 }
1174 if self.core.table.live > 0
1175 && (self.core.table.writes_live() || !self.core.table.consistent())
1176 {
1177 self.store.inner.domain.poison(FirstCause {
1178 op: Some(Op::Write),
1179 raw: failed,
1180 });
1181 self.store.wake_frontiers();
1182 }
1183 self.store.unregister_submitter(self.id);
1184 }
1185}