store_io_engine/submitter/mod.rs
1//! The engine-layer [`Submitter`] (D-065, D-080, D-098): many operations in
2//! flight from one thread, without blocking, with the same integrity rules
3//! as the blocking calls.
4//!
5//! Every operation is admitted like a blocking call (the region's gate, the
6//! store's state, the same validation), then held in a fixed table until its
7//! completion is processed by [`Submitter::reap`] or [`Submitter::wait`]:
8//!
9//! - its region's **owned pass**, so recycle and release wait for it;
10//! - for an append, its **frontier reservation**, completed only when the
11//! write transferred every byte (else the store is poisoned: the reserved
12//! range is a hole no later append may be acknowledged past);
13//! - its tag and submission sequence.
14//!
15//! A [`WriteTicket`](crate::WriteTicket) is minted only from a full
16//! completion (D-080). Queue tags are table indices plus a generation, so a
17//! completion matches exactly one operation. Nothing allocates after
18//! creation. Durable writes and barriers ([`durable`]) wait for a flush
19//! without blocking: the store's flush thread answers them (D-099).
20
21mod durable;
22
23use std::marker::PhantomData;
24use std::sync::Arc;
25
26use store_io_core::error::{Error, ErrorContext, FirstCause, NotWrittenCause, Op, OsError};
27use store_io_core::observe::Counter;
28use store_io_platform::{IoBuf, IoOp, Owner, Platform, Queue, QueueConfig};
29
30use crate::completion::{Completions, Rejected, SubmitterConfig, WriteMode};
31use crate::exec::LaneMark;
32use crate::flusher::{Inbox, SubmitterWaker, WakerOf};
33use crate::gate::{Gate, Gated, OwnedPass};
34use crate::io::{ctx, ctx_range, not_written};
35use crate::lifecycle::retired_error;
36use crate::optable::{Finish, Kind, Split};
37use crate::queue_core::Core;
38use crate::receipt::RegionPos;
39use crate::region::{AppendRegion, PageRegion};
40use crate::store::{LiveRegion, Store};
41
42/// The largest `SubmitterConfig::depth`.
43pub const MAX_DEPTH: u32 = 4096;
44
45impl Gated for LiveRegion {
46 fn gate(&self) -> &Gate {
47 &self.gate
48 }
49}
50
51/// Many reads and writes in flight from one thread (D-065, D-098).
52///
53/// Submitting never blocks: a full submitter is refused with the buffer
54/// given back ([`Rejected`], `NotWritten(QueueFull)`). Completions are
55/// handed back by [`Self::reap`] (never blocks) and [`Self::wait`]
56/// (blocks), with a [`WriteTicket`](crate::WriteTicket) for every write that
57/// completed in full. Buffers come from [`Self::take_buf`] (the store's
58/// pool, shared with its blocking calls: buffers held in flight or kept by
59/// the caller are not available to them) and come back in the completion.
60///
61/// The rules the blocking calls keep hold per operation:
62///
63/// - **An append's position is final at submission** (returned at once),
64/// but it counts as written only once its completion is processed by
65/// this submitter's `reap` or `wait`. Until then, on that region, from
66/// any thread: `sync_through` of any later append waits for it, and an
67/// append starting more than `StoreOptions::append_window` bytes past it
68/// waits (blocking calls) or is refused `NotWritten(TooManyInFlight)`
69/// (submitters). Reap your own appends before syncing or appending far
70/// past them, or the thread waits for itself.
71/// - **Recycle and release wait** for every operation of the submitter on
72/// that region until its completion is processed: a thread must reap its
73/// own submitter's operations on a region before it recycles or releases
74/// that region.
75/// - **A write that fails or transfers short after it was issued** is
76/// `DurabilityUnknown`, and the store is poisoned.
77///
78/// Dropping the submitter waits, untimed, for every operation in flight
79/// before it releases anything the device may still touch. A device that
80/// never completes an operation blocks the drop: the fail-stop choice
81/// (see `Store::oldest_outstanding` for a watchdog).
82///
83/// A submitter belongs to the thread that created it:
84///
85/// ```compile_fail,E0277
86/// fn need_send<T: Send>() {}
87/// need_send::<store_io_engine::Submitter<store_io_sim::SimPlatform>>();
88/// ```
89pub struct Submitter<P: Platform>
90where
91 P::Queue: Send,
92{
93 // Declared first, so its queue drops (after `Drop::drop` drained it)
94 // before the table releases the passes and reservations.
95 core: Core<P::Queue>,
96 /// Where the flush thread answers this submitter's flush requests.
97 inbox: Arc<Inbox<<P::Queue as Queue>::Waker>>,
98 id: u64,
99 store: Store<P>,
100 _thread: PhantomData<*const ()>,
101}
102
103impl<P: Platform> core::fmt::Debug for Submitter<P>
104where
105 P::Queue: Send,
106{
107 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
108 f.debug_struct("Submitter")
109 .field("depth", &self.core.table.depth())
110 .field("in_flight", &self.core.table.live)
111 .field("ready", &self.core.table.ready.len())
112 .finish()
113 }
114}
115
116/// The request [`Submitter::issue`] turns into an [`IoOp`].
117enum Req {
118 Write { offset: u64, buf: IoBuf },
119 Read { offset: u64, buf: IoBuf },
120}
121
122/// The first device operation of `[start, start + buf.len())`: all of it,
123/// or (format 2, across a run's end, spec ยง11) the piece up to the run's
124/// end, `buf` shortened to it, the rest following under the same row
125/// ([`Split`]). Returns its file offset.
126fn first_piece(state: &LiveRegion, start: u64, buf: &mut IoBuf) -> (u64, Option<Split>) {
127 let total = buf.len();
128 let piece = (state.map.run_left(start) as usize).min(total);
129 let split = (piece < total).then(|| {
130 let fits = buf.set_len(piece);
131 debug_assert!(fits, "a piece longer than its buffer");
132 Split { total, done: 0 }
133 });
134 (state.map.locate(start, piece as u64), split)
135}
136
137fn rejected(error: Error, buf: IoBuf) -> Rejected {
138 Rejected {
139 error,
140 buf: Some(buf),
141 }
142}
143
144/// `?` for admission: refuses with the buffer given back.
145macro_rules! admit {
146 ($e:expr, $buf:ident) => {
147 match $e {
148 Ok(v) => v,
149 Err(error) => return Err(rejected(error, $buf)),
150 }
151 };
152}
153
154impl<P: Platform> Store<P>
155where
156 P::Queue: Send,
157{
158 /// A [`Submitter`] for this thread, with its own queue of
159 /// `cfg.depth` operations (1 to [`MAX_DEPTH`]).
160 ///
161 /// # Errors
162 ///
163 /// `Io` if the platform could not create the queue.
164 pub fn submitter(&self, cfg: SubmitterConfig) -> Result<Submitter<P>, Error> {
165 let depth = cfg.depth.clamp(1, MAX_DEPTH);
166 let qcfg = QueueConfig {
167 depth,
168 parallel: true,
169 pool_threads: self.inner.opts.t3_pool_threads,
170 owner: Owner::Thread,
171 };
172 let inner = &*self.inner;
173 // The file outlives the queue: a registering backend keeps its own
174 // reference to it (`Platform::queue_registered`).
175 let queue = if inner.opts.register_file {
176 inner.platform.queue_registered(qcfg, &inner.file)
177 } else {
178 inner.platform.queue(qcfg)
179 };
180 let queue = queue.map_err(|raw| Error::Io {
181 op: Op::Queue,
182 raw,
183 ctx: ErrorContext::default(),
184 })?;
185 let n = depth as usize;
186 let mark = Arc::new(LaneMark::new());
187 let inbox = Arc::new(Inbox::new(n, queue.waker()));
188 let id = self.register_submitter(&mark);
189 Ok(Submitter {
190 core: Core::new(queue, n, mark),
191 inbox,
192 id,
193 store: self.clone(),
194 _thread: PhantomData,
195 })
196 }
197}
198
199impl<P: Platform> Submitter<P>
200where
201 P::Queue: Send,
202{
203 /// A buffer of `len` bytes from the store's pool, for one operation.
204 ///
205 /// # Errors
206 ///
207 /// `NotWritten(TooLarge)` beyond [`Self::max_buf`];
208 /// `NotWritten(PoolExhausted)` if none is free.
209 pub fn take_buf(&self, len: usize) -> Result<IoBuf, Error> {
210 if len > self.max_buf() {
211 return Err(not_written(
212 NotWrittenCause::TooLarge,
213 ErrorContext::default(),
214 ));
215 }
216 self.store.take_any(len, ErrorContext::default())
217 }
218
219 /// The largest buffer, and so the largest single operation, in bytes (a
220 /// whole number of blocks): the pool's largest class, `max_write`
221 /// rounded up to a power of two (or `max_io` without large buffers). One
222 /// OS operation (D-101); the kernel may still split it into several
223 /// device requests.
224 #[must_use]
225 pub fn max_buf(&self) -> usize {
226 self.store.max_op()
227 }
228
229 /// Operations submitted whose completions have not been handed to
230 /// `reap` or `wait` yet: I/O in flight, and durable writes and barriers
231 /// waiting for their flush.
232 #[must_use]
233 pub fn in_flight(&self) -> usize {
234 // Never more than are really pending (an operation lost to a panic
235 // in processing is no longer waited for; the store is poisoned then).
236 self.core
237 .table
238 .live
239 .min(self.core.queue.in_flight() + self.core.table.asked)
240 }
241
242 /// Ends this submitter's current or next [`Self::wait`] from another
243 /// thread (D-083), whether it waits for I/O or for a flush.
244 #[must_use]
245 pub fn waker(&self) -> SubmitterWaker<WakerOf<P>> {
246 SubmitterWaker::new(&self.inbox)
247 }
248
249 /// The checks every operation makes first: this store's region, room in
250 /// the table, and the region's gate.
251 fn admit(
252 &self,
253 store: &Store<P>,
254 state: &Arc<LiveRegion>,
255 buf: &IoBuf,
256 ) -> Result<OwnedPass<LiveRegion>, Error> {
257 let c = ctx(state);
258 if !Arc::ptr_eq(&store.inner, &self.store.inner) {
259 return Err(not_written(NotWrittenCause::ForeignPosition, c));
260 }
261 // Only this store's buffers have its alignment and whole blocks of
262 // capacity; any other could fail at the device after an append's
263 // space is reserved.
264 if !self.store.inner.pool.owns(buf) {
265 return Err(not_written(NotWrittenCause::ForeignBuffer, c));
266 }
267 if self.core.table.full() {
268 return Err(not_written(NotWrittenCause::QueueFull, c));
269 }
270 OwnedPass::enter(state).map_err(|why| retired_error(why, state))
271 }
272
273 /// Appends the buffer's bytes (any length of at least one byte) and
274 /// returns their position at once; the completion carries the ticket
275 /// (and, with [`WriteMode::Durable`], the receipt).
276 /// The append starts at a block boundary and the buffer's tail is
277 /// zero-padded in place to the next one (no copy): the buffer comes back
278 /// with its length set to the padded size.
279 ///
280 /// # Errors
281 ///
282 /// [`Rejected`], with the buffer: `NotWritten` (`QueueFull`: the
283 /// submitter is full; `TooManyInFlight`: the region's append ring is
284 /// full, see [`Submitter`]; empty, region full, read-only, not
285 /// positioned, another store's region or buffer), `Poisoned`, or
286 /// `DurabilityUnknown` if the OS refused the write after its position
287 /// was reserved (the store is then poisoned, as for a blocking append).
288 pub fn append(
289 &mut self,
290 region: &AppendRegion<P>,
291 mut buf: IoBuf,
292 mode: WriteMode,
293 tag: u64,
294 ) -> Result<RegionPos, Rejected> {
295 let state = region.state();
296 let c = ctx(state);
297 let pass = admit!(self.admit(region.store(), state, &buf), buf);
298 admit!(self.store.ready_for(state), buf);
299 let Some(f) = state.frontier.as_ref() else {
300 return Err(rejected(not_written(NotWrittenCause::NotReady, c), buf));
301 };
302 if !state.positioned.load(std::sync::atomic::Ordering::Acquire) {
303 return Err(rejected(
304 not_written(NotWrittenCause::NotPositioned, c),
305 buf,
306 ));
307 }
308 let len = buf.len();
309 if len == 0 {
310 return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
311 }
312 // Checked before the reservation, so the padding below cannot fail
313 // (the pool's capacities are whole blocks anyway).
314 let block = self.store.block() as usize;
315 if len.div_ceil(block) * block > buf.capacity() {
316 return Err(rejected(not_written(NotWrittenCause::TooLarge, c), buf));
317 }
318 let domain = &self.store.inner.domain;
319 let reserved = f
320 .reserve(len as u64)
321 .map_err(|why| match domain.poisoned() {
322 Some(first) => Error::Poisoned { first },
323 None => not_written(why, c),
324 });
325 let r = admit!(reserved, buf);
326 // Held until the row holds it: a panic before then (in `locate`, or
327 // a backend's `submit`) poisons and wakes every waiter at once,
328 // never a hole that a later barrier waits on until this submitter
329 // drops. The store is the region's (`admit` checked it is this one).
330 let held = region.store().hold_reservation(r);
331 let padded = (r.end - r.start) as usize;
332 buf.as_mut_capacity()[len..padded].fill(0);
333 let _fits = buf.set_len(padded);
334 let pos = RegionPos {
335 volume: self.store.inner.volume,
336 region: state.id,
337 generation: state.generation,
338 offset: r.start,
339 };
340 let (offset, split) = first_piece(state, r.start, &mut buf);
341 let req = Req::Write { offset, buf };
342 let durable = mode == WriteMode::Durable;
343 let issued = self.issue(req, tag, Kind::Append(r), pass, durable, split);
344 let _r = held.disarm();
345 match issued {
346 Ok(()) => Ok(pos),
347 // Past the reservation: a hole, as for a blocking append.
348 Err((raw, buf)) => Err(Rejected {
349 error: self
350 .store
351 .unknown(Op::Write, raw, ctx_range(state, r.start, r.data_end)),
352 buf,
353 }),
354 }
355 }
356
357 /// Writes the buffer (whole blocks) at `pos`; the completion carries the
358 /// ticket (and, with [`WriteMode::Durable`], the receipt).
359 ///
360 /// # Errors
361 ///
362 /// [`Rejected`], with the buffer: `NotWritten` (foreign, stale,
363 /// misaligned, out of range, submitter full, read-only, another store's
364 /// buffer), `Poisoned`, or `Io` if the OS refused it (nothing reached
365 /// the device).
366 pub fn write(
367 &mut self,
368 region: &PageRegion<P>,
369 pos: RegionPos,
370 mut buf: IoBuf,
371 mode: WriteMode,
372 tag: u64,
373 ) -> Result<(), Rejected> {
374 let state = region.state();
375 let pass = admit!(self.admit(region.store(), state, &buf), buf);
376 admit!(self.store.ready_for(state), buf);
377 let end = admit!(region.check(&pos, buf.len()), buf);
378 let c = ctx_range(state, pos.offset, end);
379 let (offset, split) = first_piece(state, pos.offset, &mut buf);
380 let req = Req::Write { offset, buf };
381 let kind = Kind::Page {
382 start: pos.offset,
383 end,
384 };
385 self.issue(req, tag, kind, pass, mode == WriteMode::Durable, split)
386 .map_err(|(raw, buf)| Rejected {
387 error: Store::<P>::refused(raw, c),
388 buf,
389 })
390 }
391
392 /// Reads into the buffer (whole blocks) from the block-aligned `offset`
393 /// of an append region; the completion carries the bytes. Reads work on
394 /// read-only and poisoned stores.
395 ///
396 /// # Errors
397 ///
398 /// [`Rejected`], with the buffer: `NotWritten` (empty, misaligned, out
399 /// of range, submitter full, another store's region or buffer), or `Io`
400 /// if the OS refused it.
401 pub fn read_append(
402 &mut self,
403 region: &AppendRegion<P>,
404 offset: u64,
405 buf: IoBuf,
406 tag: u64,
407 ) -> Result<(), Rejected> {
408 self.read(region.store(), region.state(), offset, buf, tag)
409 }
410
411 /// Reads into the buffer (whole blocks) at `pos` of a page region; the
412 /// completion carries the bytes.
413 ///
414 /// # Errors
415 ///
416 /// As [`Self::read_append`], and `NotWritten(ForeignPosition |
417 /// StaleGeneration)` for a position of another region or generation.
418 pub fn read_page(
419 &mut self,
420 region: &PageRegion<P>,
421 pos: RegionPos,
422 buf: IoBuf,
423 tag: u64,
424 ) -> Result<(), Rejected> {
425 let state = region.state();
426 let c = ctx(state);
427 if pos.volume != self.store.inner.volume || pos.region != state.id {
428 return Err(rejected(
429 not_written(NotWrittenCause::ForeignPosition, c),
430 buf,
431 ));
432 }
433 if pos.generation != state.generation {
434 return Err(rejected(
435 not_written(NotWrittenCause::StaleGeneration, c),
436 buf,
437 ));
438 }
439 self.read(region.store(), state, pos.offset, buf, tag)
440 }
441
442 fn read(
443 &mut self,
444 store: &Store<P>,
445 state: &Arc<LiveRegion>,
446 offset: u64,
447 mut buf: IoBuf,
448 tag: u64,
449 ) -> Result<(), Rejected> {
450 let pass = admit!(self.admit(store, state, &buf), buf);
451 let len = buf.len() as u64;
452 let block = self.store.block();
453 let c = ctx_range(state, offset, offset.saturating_add(len));
454 if len == 0 {
455 return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
456 }
457 if offset % block != 0 || len % block != 0 {
458 return Err(rejected(not_written(NotWrittenCause::Misaligned, c), buf));
459 }
460 if offset.checked_add(len).is_none_or(|e| e > state.map.len()) {
461 return Err(rejected(not_written(NotWrittenCause::OutOfBounds, c), buf));
462 }
463 let (at, split) = first_piece(state, offset, &mut buf);
464 let req = Req::Read { offset: at, buf };
465 let kind = Kind::Read {
466 start: offset,
467 end: offset + len,
468 };
469 self.issue(req, tag, kind, pass, false, split)
470 .map_err(|(raw, buf)| Rejected {
471 error: match raw {
472 Some(raw) => Error::Io {
473 op: Op::Read,
474 raw,
475 ctx: c,
476 },
477 None => not_written(NotWrittenCause::QueueFull, c),
478 },
479 buf,
480 })
481 }
482
483 /// Submits an admitted operation into a free row. On a refusal the row
484 /// stays free and the pass is released.
485 fn issue(
486 &mut self,
487 req: Req,
488 tag: u64,
489 kind: Kind,
490 pass: OwnedPass<LiveRegion>,
491 durable: bool,
492 split: Option<Split>,
493 ) -> Result<(), (Option<OsError>, Option<IoBuf>)> {
494 // A refusal gives the caller's buffer back whole (a split's first
495 // piece is its front: nothing was moved).
496 let whole = |mut buf: IoBuf| {
497 if let Some(s) = split {
498 let fits = buf.set_len(s.total);
499 debug_assert!(fits, "a split buffer shorter than its operation");
500 }
501 buf
502 };
503 // `admit` checked there is room: a free row always exists here.
504 let Some((index, qtag)) = self.core.table.claim() else {
505 let buf = match req {
506 Req::Write { buf, .. } | Req::Read { buf, .. } => buf,
507 };
508 return Err((None, Some(whole(buf))));
509 };
510 let inner = &*self.store.inner;
511 let file = &inner.file;
512 let (op, write) = match req {
513 Req::Write { offset, buf } => {
514 let bytes = buf.len() as u64;
515 let op = IoOp::Write {
516 file,
517 offset,
518 buf,
519 dsync: inner.policy.dsync,
520 };
521 (op, Some(bytes))
522 }
523 Req::Read { offset, buf } => (IoOp::Read { file, offset, buf }, None),
524 };
525 // Recorded (and the busy mark published) before the submission: a
526 // queue may do the I/O inside `submit`.
527 self.core.table.start(index, tag, kind, Some(pass), durable);
528 if let Some(s) = split {
529 self.core.table.set_split(index, s);
530 }
531 if let Err(r) = self.core.queue.submit(op, qtag) {
532 self.core.table.abort(index);
533 if r.raw.is_none() {
534 inner.observer.add(Counter::Refused, 1);
535 }
536 return Err((r.raw, r.buf.map(whole)));
537 }
538 if let Some(bytes) = write {
539 inner.observer.add(Counter::Writes, 1);
540 inner.observer.add(Counter::WriteBytes, bytes);
541 }
542 Ok(())
543 }
544
545 /// Hands back every completion that has arrived, without blocking.
546 #[must_use = "completions not taken stay with the submitter and count against its depth"]
547 pub fn reap(&mut self) -> Completions<'_> {
548 let _added = self.core.queue.reap(&mut self.core.raw);
549 self.process();
550 Completions {
551 ready: &mut self.core.table.ready,
552 }
553 }
554
555 /// Blocks until at least `min` completions are ready to hand back (no
556 /// more than are in flight or ready), or the submitter's waker fires,
557 /// then hands back every one that has arrived.
558 ///
559 /// # Errors
560 ///
561 /// `Io` if waiting itself failed: the operations stay in flight and a
562 /// later `reap` or `wait` collects them (a failed wait is never a
563 /// verdict on the media, D-097).
564 #[must_use = "completions not taken stay with the submitter and count against its depth"]
565 pub fn wait(&mut self, min: usize) -> Result<Completions<'_>, Error> {
566 let want = min.min(self.core.table.live + self.core.table.ready.len());
567 if self.core.table.ready.len() >= want {
568 let _added = self.core.queue.reap(&mut self.core.raw);
569 self.process();
570 }
571 while self.core.table.ready.len() < want {
572 // Only the caller's waker ends a wait early: a wake left in the
573 // queue by the flush thread just loops.
574 if self.inbox.take_woken() {
575 break;
576 }
577 let in_queue = self.core.queue.in_flight();
578 if in_queue > 0 {
579 let need = (want - self.core.table.ready.len()).min(in_queue);
580 let waited = if self.inbox.block_in_queue() {
581 let waited = self.core.queue.wait(need, &mut self.core.raw);
582 self.inbox.unblock();
583 waited
584 } else {
585 // A flush outcome or a wake arrived: no blocking.
586 Ok(self.core.queue.reap(&mut self.core.raw))
587 };
588 // Whatever arrived is processed, even if the wait then failed.
589 self.process();
590 if let Err(raw) = waited {
591 return Err(Error::Io {
592 op: if self.core.table.writes_live() {
593 Op::Write
594 } else {
595 Op::Read
596 },
597 raw,
598 ctx: ErrorContext::default(),
599 });
600 }
601 } else if self.core.table.asked > 0 {
602 self.inbox.wait(true);
603 self.process();
604 } else {
605 break;
606 }
607 }
608 Ok(Completions {
609 ready: &mut self.core.table.ready,
610 })
611 }
612
613 /// Processes every raw completion collected.
614 fn process(&mut self) {
615 let _unwind = PoisonOnUnwind::new(&self.store);
616 for c in self.core.raw.drain() {
617 // Only a write needing a flush asks (see `Table::finish`).
618 let mut finished = self.core.table.finish(&self.store, c.tag, c.buf, c.result);
619 while let Finish::Next {
620 qtag,
621 offset,
622 buf,
623 write,
624 } = finished
625 {
626 let inner = &*self.store.inner;
627 let file = &inner.file;
628 let bytes = buf.len() as u64;
629 let op = if write {
630 IoOp::Write {
631 file,
632 offset,
633 buf,
634 dsync: inner.policy.dsync,
635 }
636 } else {
637 IoOp::Read { file, offset, buf }
638 };
639 finished = match self.core.queue.submit(op, qtag) {
640 Ok(()) => {
641 if write {
642 inner.observer.add(Counter::Writes, 1);
643 inner.observer.add(Counter::WriteBytes, bytes);
644 }
645 Finish::Done
646 }
647 Err(r) => {
648 if r.raw.is_none() {
649 inner.observer.add(Counter::Refused, 1);
650 }
651 self.core
652 .table
653 .refused_next(&self.store, qtag, r.buf, r.raw)
654 }
655 };
656 }
657 if let Finish::Flush(qtag, need) = finished {
658 self.core.table.asks.push((qtag, need));
659 }
660 }
661 // Asked once every completion of the pass is finished, the largest
662 // need first, so one flush can cover them all. Taken one at a time:
663 // after a panic the rest stay to be asked for, in the same buffer.
664 while let Some((qtag, need)) = self.core.table.asks.pop() {
665 durable::ask(
666 &mut self.core.table,
667 &self.store,
668 &self.inbox,
669 qtag,
670 need,
671 true,
672 );
673 }
674 // Flush outcomes too, under the same guard.
675 while self.core.table.asked > 0 {
676 let Some((qtag, outcome)) = self.inbox.pop() else {
677 break;
678 };
679 self.core.table.asked -= 1;
680 self.core.table.flushed(&self.store, qtag, outcome);
681 }
682 }
683}
684
685/// Poisons the store if a panic unwinds through the processing of
686/// completions: an operation taken from its row but not settled may be an
687/// append whose reserved range is then never completed.
688struct PoisonOnUnwind<'a, P: Platform>
689where
690 P::Queue: Send,
691{
692 store: &'a Store<P>,
693 /// Already unwinding when created (a drop during another panic): not
694 /// this processing's panic.
695 was: bool,
696}
697
698impl<'a, P: Platform> PoisonOnUnwind<'a, P>
699where
700 P::Queue: Send,
701{
702 fn new(store: &'a Store<P>) -> Self {
703 Self {
704 store,
705 was: std::thread::panicking(),
706 }
707 }
708}
709
710impl<P: Platform> Drop for PoisonOnUnwind<'_, P>
711where
712 P::Queue: Send,
713{
714 fn drop(&mut self) {
715 if !self.was && std::thread::panicking() {
716 self.store.inner.domain.poison(FirstCause {
717 op: Some(Op::Write),
718 raw: None,
719 });
720 self.store.wake_frontiers();
721 }
722 }
723}
724
725impl<P: Platform> Drop for Submitter<P>
726where
727 P::Queue: Send,
728{
729 /// Waits for every operation in flight, untimed (TYP-05), so nothing the
730 /// device may still touch is released. If waiting fails, the queue's own
731 /// drop keeps what the kernel owns, and the store is poisoned when a
732 /// write was in flight: its range may never be written, and nothing of
733 /// its generation may be recycled while it could still land. The same
734 /// if a panic interrupted the processing of a completion (an operation
735 /// left unaccounted may have been an append).
736 fn drop(&mut self) {
737 let mut failed = None;
738 // First every I/O (a durable write's completion may ask for a
739 // flush)...
740 while self.core.table.live > 0 && self.core.queue.in_flight() > 0 {
741 match self
742 .core
743 .queue
744 .wait(self.core.queue.in_flight(), &mut self.core.raw)
745 {
746 Ok(_) => self.process(),
747 Err(raw) => {
748 failed = Some(raw);
749 break;
750 }
751 }
752 self.core.table.ready.clear();
753 }
754 // ... with whatever the queue collected before a failed wait...
755 let _added = self.core.queue.reap(&mut self.core.raw);
756 self.process();
757 // ... then every flush asked for: the flush thread answers every
758 // request it accepted.
759 while self.core.table.asked > 0 {
760 self.inbox.wait(false);
761 self.process();
762 }
763 if self.core.table.live > 0
764 && (self.core.table.writes_live() || !self.core.table.consistent())
765 {
766 self.store.inner.domain.poison(FirstCause {
767 op: Some(Op::Write),
768 raw: failed,
769 });
770 self.store.wake_frontiers();
771 }
772 self.store.unregister_submitter(self.id);
773 }
774}