Skip to main content

store_io_engine/
stream.rs

1//! Read streams (PLANNING R5.5): a region's bytes as owned chunks, in
2//! order, from a queue of the stream's own.
3//!
4//! A [`ReadStream`] keeps several reads in flight on a private queue
5//! (`Owner::Thread`, like a `Submitter`'s), so it holds no lane a commit
6//! needs and none between [`Iterator::next`] calls: the caller may use the
7//! store while it iterates. Each item is a [`Chunk`] that owns a pool
8//! buffer, or an exact `Corruption { MediaError, range }` for bytes the
9//! device cannot read, after which the stream goes on.
10//!
11//! **No pass across the I/O.** The stream holds no gate pass, so a caller's
12//! recycle or release never waits on it. Its region's gate is checked
13//! (`Gate::live`, a read-modify-write) before each read is submitted and
14//! again in `next()` once the read's completion is seen and before anything
15//! of it is delivered, counted or classified. A gate still live then means
16//! no retire began before the read completed, or one began and was restored
17//! before anything on disk changed, so no chunk holds bytes of a newer
18//! generation and no error of a released range is reported as corruption.
19//! A media error is bisected on the stream's own idle queue into one
20//! scratch buffer, each good result copied to its place in the piece's
21//! buffer; its items are queued only after the gate holds once all its
22//! reads have completed.
23//!
24//! **Buffers.** A stream takes `depth + lend + 1` buffers when it is made,
25//! under the pool's background cap (half of each class: a stream is a bulk
26//! reader, so the other half stays for commits): up to `depth` reads in
27//! flight, up to `lend` chunks held by its caller, and the bisection's
28//! scratch. A dropped [`Chunk`] gives its buffer back to the stream, or to
29//! the pool once the stream has ended, from any thread.
30
31use std::cell::RefCell;
32use std::collections::VecDeque;
33use std::marker::PhantomData;
34use std::ops::Range;
35use std::rc::{Rc, Weak};
36use std::sync::{Arc, PoisonError};
37
38#[cfg(loom)]
39use loom::sync::{Mutex, MutexGuard};
40#[cfg(not(loom))]
41use std::sync::{Mutex, MutexGuard};
42
43use store_io_core::IoClass;
44use store_io_core::errno::is_media_error;
45use store_io_core::error::{CorruptionKind, Error, ErrorContext, NotWrittenCause, Op, OsError};
46use store_io_core::observe::Counter;
47use store_io_platform::{IoBuf, IoOp, Platform, RawResult};
48
49use crate::exec::{Lane, LaneMark, Ran, owned_lane, run};
50use crate::io::{ctx_range, not_written};
51use crate::lifecycle::retired_error;
52use crate::pace::{Held, Pace, Registered};
53use crate::qos::Collect;
54use crate::store::{LiveRegion, Store};
55use crate::submitter::MAX_DEPTH;
56use crate::test_hook::{self, Step};
57
58/// Reads in flight by default, at most (fewer on a pool with fewer
59/// buffers free under the cap).
60const DEFAULT_DEPTH: usize = 8;
61
62/// How a [`ReadStream`] reads. `depth`: reads in flight, 1 to
63/// [`MAX_DEPTH`](crate::MAX_DEPTH); `lend`: chunks its caller may hold at
64/// once, at least 1. With neither set, the stream takes up to 8 of each
65/// (fewer, and shallower, when fewer buffers are free under the pool's
66/// background cap); with either set, the other defaults to it and the
67/// stream takes exactly `depth + lend + 1` buffers or is refused. `class`:
68/// [`IoClass::Background`] (the default: `background_io` requests, paced
69/// while the device's commits are active) or [`IoClass::Read`] (`max_io`
70/// requests, unpaced: a foreground read its caller waits on).
71#[derive(Clone, Copy, Debug, PartialEq, Eq)]
72#[non_exhaustive]
73pub struct StreamOptions {
74    depth: Option<usize>,
75    lend: Option<usize>,
76    class: IoClass,
77}
78
79impl Default for StreamOptions {
80    fn default() -> Self {
81        Self {
82            depth: None,
83            lend: None,
84            class: IoClass::Background,
85        }
86    }
87}
88
89impl StreamOptions {
90    /// Reads in flight at most.
91    #[must_use]
92    pub fn depth(mut self, depth: usize) -> Self {
93        self.depth = Some(depth);
94        self
95    }
96
97    /// Chunks the caller may hold at once.
98    #[must_use]
99    pub fn lend(mut self, lend: usize) -> Self {
100        self.lend = Some(lend);
101        self
102    }
103
104    /// The class of its reads: `Background` or `Read`.
105    #[must_use]
106    pub fn class(mut self, class: IoClass) -> Self {
107        self.class = class;
108        self
109    }
110}
111
112/// A stream's buffers not in flight and not lent, and how many are lent.
113/// Shared with its chunks, which give their buffers back from any thread:
114/// kept while the stream lives, handed back to the pool once it is closed.
115/// `closed` is read and a buffer pushed under one lock, so no buffer is
116/// left on a list already drained.
117pub(crate) struct FreeList<B> {
118    closed: bool,
119    bufs: Vec<B>,
120    lent: usize,
121}
122
123impl<B> FreeList<B> {
124    pub(crate) fn new(bufs: Vec<B>) -> Self {
125        Self {
126            closed: false,
127            bufs,
128            lent: 0,
129        }
130    }
131
132    /// A lent buffer back: kept while open; returned, to be dropped outside
133    /// the lock, once closed.
134    pub(crate) fn give_back(&mut self, b: B) -> Option<B> {
135        self.lent = self.lent.saturating_sub(1);
136        if self.closed {
137            Some(b)
138        } else {
139            self.bufs.push(b);
140            None
141        }
142    }
143
144    /// Closes the list: every buffer on it, to be dropped outside the lock.
145    pub(crate) fn close(&mut self) -> Vec<B> {
146        self.closed = true;
147        std::mem::take(&mut self.bufs)
148    }
149}
150
151fn lock<B>(m: &Mutex<FreeList<B>>) -> MutexGuard<'_, FreeList<B>> {
152    m.lock().unwrap_or_else(PoisonError::into_inner)
153}
154
155/// Bytes of a region, read by a [`ReadStream`]: it owns a pool buffer,
156/// given back when the chunk drops (to its stream while the stream lives,
157/// else to the pool). `Send`: a worker thread may hold and drop it.
158pub struct Chunk {
159    buf: Option<IoBuf>,
160    at: usize,
161    len: usize,
162    offset: u64,
163    range: usize,
164    home: Arc<Mutex<FreeList<IoBuf>>>,
165}
166
167impl Chunk {
168    /// The region offset of `bytes()[0]` (for an export's region copied
169    /// in full, its offset in the region's stream: the sum of the earlier
170    /// container ranges' lengths and its place in its own).
171    #[must_use]
172    pub fn offset(&self) -> u64 {
173        self.offset
174    }
175
176    /// Which of the stream's ranges it is of: its index in
177    /// [`Store::prefetch`](crate::Store::prefetch)'s `ranges` (0 for a
178    /// region's own stream), so overlapping ranges read twice are told
179    /// apart.
180    #[must_use]
181    pub fn range(&self) -> usize {
182        self.range
183    }
184
185    /// The bytes.
186    #[must_use]
187    pub fn bytes(&self) -> &[u8] {
188        match &self.buf {
189            Some(b) => &b.as_slice()[self.at..self.at + self.len],
190            None => &[],
191        }
192    }
193}
194
195impl core::fmt::Debug for Chunk {
196    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
197        f.debug_struct("Chunk")
198            .field("offset", &self.offset)
199            .field("len", &self.len)
200            .field("range", &self.range)
201            .finish_non_exhaustive()
202    }
203}
204
205impl Drop for Chunk {
206    fn drop(&mut self) {
207        if let Some(b) = self.buf.take() {
208            // Dropped (to the pool, its background count released) outside
209            // the lock once the stream has ended.
210            let back = lock(&self.home).give_back(b);
211            drop(back);
212        }
213    }
214}
215
216/// A read in flight, and its completion once it arrives.
217struct Read {
218    /// Its part (an index into the stream's parts).
219    part: usize,
220    start: u64,
221    len: usize,
222    done: Option<(RawResult<usize>, Option<IoBuf>)>,
223    /// A `Background` stream's charge for the read: released when its
224    /// completion is collected (or the stream ends).
225    charge: Option<Held>,
226}
227
228/// What a bisected piece comes to, in order: runs of its buffer and the
229/// ranges that could not be read.
230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231enum Item {
232    /// `len` bytes of the piece's buffer from `at`, region offset `start`;
233    /// `last`: the piece's buffer itself is lent with it.
234    Run {
235        part: usize,
236        start: u64,
237        at: usize,
238        len: usize,
239        last: bool,
240    },
241    /// `[start, end)` could not be read.
242    Bad { part: usize, start: u64, end: u64 },
243}
244
245/// One range of a stream: its region (owned, no pass), the bytes asked for,
246/// its index in the caller's ranges, and where its bytes lie.
247struct Part {
248    r: Arc<LiveRegion>,
249    want: Range<u64>,
250    index: usize,
251    loc: Locator,
252}
253
254/// Where a part's bytes lie in the container.
255pub(crate) enum Locator {
256    /// Through its region's map: a stream offset is a region offset.
257    Map,
258    /// Container ranges `(offset, len)`, in stream order (an export's
259    /// region of a stale page, copied in full: its capacity's runs, then
260    /// its extra extent): a stream offset is the sum of the earlier ranges'
261    /// lengths and the place in its own. Never through the map: the extra
262    /// extent is outside it.
263    Ranges(Vec<(u64, u64)>),
264}
265
266impl Part {
267    /// The bytes from stream offset `at` to the end of the device run that
268    /// holds it (0: none; a backstop the refill refuses).
269    fn run_left(&self, at: u64) -> u64 {
270        match &self.loc {
271            Locator::Map => self.r.map.run_left(at),
272            Locator::Ranges(ranges) => {
273                let mut base = 0u64;
274                for &(_, len) in ranges {
275                    if at < base + len {
276                        return base + len - at;
277                    }
278                    base += len;
279                }
280                0
281            }
282        }
283    }
284
285    /// The container offset of stream offset `at` (`len` bytes from it lie
286    /// in one run: [`Self::run_left`]).
287    fn locate(&self, at: u64, len: u64) -> u64 {
288        match &self.loc {
289            Locator::Map => self.r.map.locate(at, len),
290            Locator::Ranges(ranges) => {
291                let mut base = 0u64;
292                for &(file, n) in ranges {
293                    if at < base + n {
294                        return file + (at - base);
295                    }
296                    base += n;
297                }
298                u64::MAX
299            }
300        }
301    }
302}
303
304/// A region's bytes in order, as owned [`Chunk`]s, with an exact
305/// `Corruption { MediaError, range }` item for each range the device cannot
306/// read (the stream goes on after it). Made by `read_stream` on an
307/// [`AppendRegion`](crate::AppendRegion) or a
308/// [`PageRegion`](crate::PageRegion).
309///
310/// Its queue is the thread's, so a `ReadStream` is not `Send`:
311///
312/// ```compile_fail
313/// fn assert_send<T: Send>() {}
314/// assert_send::<store_io_engine::ReadStream<store_io_sim::SimPlatform>>();
315/// ```
316///
317/// Ended by a retire of its region (seen at any check: `StaleGeneration`
318/// if recycled, `NotReady` if released; the caller restarts it), by an
319/// error, or at its end. A `next()` while every one of its `lend` chunks is
320/// still held returns `NotWritten(TooManyInFlight)` and the stream goes on:
321/// drop or give back a chunk, then call it again. Dropping the stream, or
322/// [`Self::cancel`], submits nothing more and waits for the reads in
323/// flight. The stream holds a clone of its store, so the store's clean
324/// close waits until it is dropped. A page region's chunks are no
325/// snapshot: a write made while streaming may show in them, or be torn
326/// across a chunk, as in a scan. A `Background` stream between `next()`
327/// calls keeps the charges of reads completed but not yet collected, which
328/// count against its device group's pacing budget. Where a queue is lost
329/// with reads in flight (a failed wait), their buffers never return to the
330/// pool, lowering their class's background cap for the process's life.
331pub struct ReadStream<P: Platform>
332where
333    P::Queue: Send,
334{
335    /// Its whole state, shared (weakly) with its registration as a
336    /// collector of its call's record: a `Background` stream's reads hold
337    /// the call's charges between `next()` calls, which another part of
338    /// the call on this thread collects instead of waiting on them.
339    core: Rc<RefCell<Core<P>>>,
340    _part: Option<Registered>,
341}
342
343/// A read stream's state, in one cell: a collector borrows it whole, so a
344/// collection leaves its lane, its ring and its count of reads in flight
345/// consistent.
346struct Core<P: Platform>
347where
348    P::Queue: Send,
349{
350    // Declared first, so its queue drops (after `Drop::drop` ended the
351    // stream) before the store clone, as a `Submitter`'s does.
352    /// `None`: an empty stream (no queue, no buffers).
353    lane: Option<Lane<P::Queue>>,
354    store: Store<P>,
355    /// Its ranges, in order (none empty), and the one being read.
356    parts: Vec<Part>,
357    cur: usize,
358    /// Its busy mark's id among the store's submitters.
359    mark: Option<u64>,
360    ring: Vec<Option<Read>>,
361    submitted: u64,
362    delivered: u64,
363    in_flight: usize,
364    /// A bisected piece's items not yet delivered, and its buffer.
365    pending: VecDeque<Item>,
366    piece: Option<IoBuf>,
367    scratch: Option<IoBuf>,
368    home: Arc<Mutex<FreeList<IoBuf>>>,
369    lend: usize,
370    /// The current part's next read, and the block-aligned end of its span.
371    next: u64,
372    end: u64,
373    req: usize,
374    pace: Option<Pace>,
375    ended: bool,
376    /// `finish` ran (once: a failed wait is never waited on again).
377    finished: bool,
378    wait_failed: Option<OsError>,
379    /// A bisection's reads, counted once its check holds.
380    bisected: (u64, u64),
381    /// A collection for another part failed (sticky): its error and the
382    /// stream offset of the oldest read lost.
383    collect_failed: Option<(OsError, u64)>,
384    _thread: PhantomData<*const ()>,
385}
386
387impl<P: Platform> core::fmt::Debug for ReadStream<P>
388where
389    P::Queue: Send,
390{
391    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
392        let mut d = f.debug_struct("ReadStream");
393        if let Ok(c) = self.core.try_borrow() {
394            let _d = d
395                .field("parts", &c.parts.len())
396                .field("cur", &c.cur)
397                .field("next", &c.next)
398                .field("ended", &c.ended);
399        }
400        d.finish_non_exhaustive()
401    }
402}
403
404/// An item that ends nothing: a chunk, or an error item the stream goes on
405/// after.
406type Next = Option<Result<Chunk, Error>>;
407
408/// One range for [`Store::prefetch`]: a region's bytes, checked when made
409/// by [`AppendRegion::stream_range`](crate::AppendRegion::stream_range) or
410/// [`PageRegion::stream_range`](crate::PageRegion::stream_range) (in
411/// bounds, an append region's clamped to its durable prefix then).
412pub struct StreamRange<'a, P: Platform> {
413    store: &'a Store<P>,
414    r: Arc<LiveRegion>,
415    want: Range<u64>,
416}
417
418impl<'a, P: Platform> StreamRange<'a, P> {
419    pub(crate) fn new(store: &'a Store<P>, r: &Arc<LiveRegion>, want: Range<u64>) -> Self {
420        Self {
421            store,
422            r: Arc::clone(r),
423            want,
424        }
425    }
426}
427
428impl<P: Platform> core::fmt::Debug for StreamRange<'_, P> {
429    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
430        f.debug_struct("StreamRange")
431            .field("region", &self.r.id)
432            .field("want", &self.want)
433            .finish_non_exhaustive()
434    }
435}
436
437impl<P: Platform> Store<P>
438where
439    P::Queue: Send,
440{
441    /// The bytes of several ranges, of this store's regions, as one
442    /// [`ReadStream`] on one queue: each range in the order given, each in
443    /// offset order ([`Chunk::range`] says which; overlapping ranges are
444    /// read twice; no range, an empty stream). An append region's range was
445    /// clamped to its durable prefix when it was made. Each region's gate is
446    /// checked again here and as the stream reads it: a retire of a region
447    /// ends the stream before any more of that region's reads are
448    /// delivered. An error tied to no read (`TooManyInFlight`, a failed
449    /// wait, `Interrupted`) names the range being read ahead.
450    ///
451    /// # Errors
452    ///
453    /// `NotWritten`: `ForeignPosition` (a range of another store, or of
454    /// another open of this one), `StaleGeneration` or `NotReady` (a region
455    /// recycled or released), `PoolExhausted`, `InvalidOptions`; `Io` if
456    /// its queue could not be made.
457    pub fn prefetch(
458        &self,
459        ranges: &[StreamRange<'_, P>],
460        opts: StreamOptions,
461    ) -> Result<ReadStream<P>, Error> {
462        for sr in ranges {
463            if !Arc::ptr_eq(&sr.store.inner, &self.inner) {
464                return Err(not_written(
465                    NotWrittenCause::ForeignPosition,
466                    ctx_range(&sr.r, sr.want.start, sr.want.end),
467                ));
468            }
469            sr.r.gate.live().map_err(|why| retired_error(why, &sr.r))?;
470        }
471        let parts = ranges
472            .iter()
473            .enumerate()
474            .map(|(i, sr)| (Arc::clone(&sr.r), sr.want.clone(), i))
475            .collect();
476        self.read_stream_of(parts, opts)
477    }
478}
479
480impl<P: Platform> Store<P>
481where
482    P::Queue: Send,
483{
484    /// A stream over `parts`, each a region, the bytes asked of it and its
485    /// index in the caller's ranges (checked by the caller: in bounds, an
486    /// append region's clamped to its durable prefix, the gate live).
487    pub(crate) fn read_stream_of(
488        &self,
489        parts: Vec<(Arc<LiveRegion>, Range<u64>, usize)>,
490        opts: StreamOptions,
491    ) -> Result<ReadStream<P>, Error> {
492        // A `Background` stream is a call of its own (R5.4).
493        let pace = match opts.class {
494            IoClass::Read => None,
495            _ => self.inner.domain.new_pace(),
496        };
497        self.stream_of(parts, opts, pace, false)
498    }
499
500    /// An export's stream (R5.6): `Background`, paced by the export's call
501    /// (`pace`, its record on this store's device group), its buffers and
502    /// queue taken now with no part yet: each region is added at its turn
503    /// ([`ReadStream::add_part`]), so `lend` is known before the sink's
504    /// `begin`.
505    pub(crate) fn export_stream(
506        &self,
507        depth: Option<usize>,
508        pace: Option<Pace>,
509    ) -> Result<ReadStream<P>, Error> {
510        let mut opts = StreamOptions::default();
511        if let Some(d) = depth {
512            opts = opts.depth(d);
513        }
514        self.stream_of(Vec::new(), opts, pace, true)
515    }
516
517    /// A stream over `parts` paced by `pace`; `open`: its queue and buffers
518    /// taken even with no part (an export's, added later).
519    fn stream_of(
520        &self,
521        parts: Vec<(Arc<LiveRegion>, Range<u64>, usize)>,
522        opts: StreamOptions,
523        pace: Option<Pace>,
524        open: bool,
525    ) -> Result<ReadStream<P>, Error> {
526        let none = ErrorContext::default();
527        if !matches!(opts.class, IoClass::Background | IoClass::Read) {
528            return Err(not_written(NotWrittenCause::InvalidOptions, none));
529        }
530        let explicit = match (opts.depth, opts.lend) {
531            (None, None) => None,
532            (d, l) => {
533                let depth = d.or(l).unwrap_or(1);
534                let lend = l.or(d).unwrap_or(1);
535                if depth == 0 || depth > MAX_DEPTH as usize || lend == 0 {
536                    return Err(not_written(NotWrittenCause::InvalidOptions, none));
537                }
538                Some((depth, lend))
539            }
540        };
541        let home = Arc::new(Mutex::new(FreeList::new(Vec::new())));
542        let block = self.block();
543        // The error context of a refusal at the call: the first range's.
544        let c = parts
545            .first()
546            .map_or(none, |(r, w, _)| ctx_range(r, w.start, w.end));
547        let parts: Vec<Part> = parts
548            .into_iter()
549            .filter(|(_, want, _)| !want.is_empty())
550            .map(|(r, want, index)| Part {
551                r,
552                want,
553                index,
554                loc: Locator::Map,
555            })
556            .collect();
557        let (next, end) = parts.first().map_or((0, 0), |p| {
558            (
559                p.want.start / block * block,
560                p.want.end.div_ceil(block) * block,
561            )
562        });
563        let mut s = Core {
564            lane: None,
565            store: self.clone(),
566            parts,
567            cur: 0,
568            mark: None,
569            ring: Vec::new(),
570            submitted: 0,
571            delivered: 0,
572            in_flight: 0,
573            pending: VecDeque::new(),
574            piece: None,
575            scratch: None,
576            home,
577            lend: 0,
578            next,
579            end,
580            req: 0,
581            pace: None,
582            ended: false,
583            finished: false,
584            wait_failed: None,
585            bisected: (0, 0),
586            collect_failed: None,
587            _thread: PhantomData,
588        };
589        let req = match (opts.class, &pace) {
590            (IoClass::Read, _) => self.chunk(),
591            (_, Some(p)) => p.piece().min(self.chunk()),
592            (_, None) => (self.inner.qos.background_io() as usize).min(self.chunk()),
593        };
594        // From the class that holds a request, under its background cap.
595        let pool = &self.inner.pool;
596        let class = pool.class_len(req).unwrap_or(req);
597        let cap = pool
598            .background_holdings()
599            .iter()
600            .find(|(size, _, _)| *size == class)
601            .map_or(0, |&(_, _, cap)| cap as usize);
602        let exhausted = || not_written(NotWrittenCause::PoolExhausted, c);
603        // Explicit options over the cap are refused, an empty range's too.
604        let explicit = match explicit {
605            Some((depth, lend)) => match depth.checked_add(lend).and_then(|n| n.checked_add(1)) {
606                Some(n) if n <= cap => Some((depth, lend, n)),
607                _ => return Err(not_written(NotWrittenCause::InvalidOptions, none)),
608            },
609            None => None,
610        };
611        if s.parts.is_empty() && !open {
612            // No queue, no buffers.
613            s.ended = true;
614            s.finished = true;
615            return Ok(ReadStream::new(s));
616        }
617        let (depth, lend, mut bufs) = match explicit {
618            Some((depth, lend, n)) => {
619                let bufs: Vec<IoBuf> = (0..n)
620                    .map_while(|_| pool.take_capped(req, class).ok())
621                    .collect();
622                if bufs.len() < n {
623                    return Err(exhausted());
624                }
625                (depth, lend, bufs)
626            }
627            None => {
628                let most = DEFAULT_DEPTH.min(cap.saturating_sub(1) / 2) * 2 + 1;
629                let bufs: Vec<IoBuf> = (0..most)
630                    .map_while(|_| pool.take_capped(req, class).ok())
631                    .collect();
632                if bufs.len() < 3 {
633                    return Err(exhausted());
634                }
635                let depth = (bufs.len() - 1) / 2;
636                (depth, bufs.len() - 1 - depth, bufs)
637            }
638        };
639        let queue = self.thread_queue(depth as u32)?;
640        let mark = Arc::new(LaneMark::new());
641        s.lane = Some(owned_lane(queue, &mark, depth));
642        s.mark = Some(self.register_submitter(&mark));
643        s.scratch = bufs.pop();
644        lock(&s.home).bufs = bufs;
645        s.ring = (0..depth).map(|_| None).collect();
646        s.lend = lend;
647        s.req = req;
648        s.pace = pace;
649        Ok(ReadStream::new(s))
650    }
651}
652
653impl<P: Platform> ReadStream<P>
654where
655    P::Queue: Send,
656{
657    /// The handle over `core`, registered as a collector of its call's
658    /// record when it is paced.
659    fn new(core: Core<P>) -> Self {
660        let pace = core.pace.clone();
661        let core = Rc::new(RefCell::new(core));
662        let _part = pace.map(|p| {
663            let weak: Weak<RefCell<Core<P>>> = Rc::downgrade(&core);
664            p.register(weak)
665        });
666        Self { core, _part }
667    }
668
669    /// Stops the stream: nothing more is submitted, the reads in flight are
670    /// waited for, and every buffer not lent goes back to the pool at once
671    /// (chunks still held go back as they drop).
672    ///
673    /// # Errors
674    ///
675    /// `Io` if waiting for the reads in flight failed: their buffers stay
676    /// with the queue, dropped with it.
677    pub fn cancel(self) -> Result<(), Error> {
678        let mut c = self.core.borrow_mut();
679        c.finish();
680        match c.wait_failed.take() {
681            Some(raw) => Err(c.read_error(raw)),
682            None => Ok(()),
683        }
684    }
685
686    /// Adds a part (an export's region at its turn): `want` of `r`, read
687    /// as `loc` places it; nothing for an empty `want`. The caller reads
688    /// every byte of the earlier parts before calling `next()` past them
689    /// (a stream with no part left ends).
690    ///
691    /// # Errors
692    ///
693    /// `NotWritten(Interrupted)` once the stream has ended.
694    pub(crate) fn add_part(
695        &self,
696        r: &Arc<LiveRegion>,
697        want: Range<u64>,
698        loc: Locator,
699    ) -> Result<(), Error> {
700        let mut c = self.core.borrow_mut();
701        if c.ended {
702            return Err(not_written(
703                NotWrittenCause::Interrupted,
704                ctx_range(r, want.start, want.end),
705            ));
706        }
707        if want.is_empty() {
708            return Ok(());
709        }
710        let block = c.store.block();
711        let first = c.parts.is_empty();
712        c.parts.push(Part {
713            r: Arc::clone(r),
714            want: want.clone(),
715            index: 0,
716            loc,
717        });
718        // The first part is the current one at once (`more` moves past a
719        // part only to a next one).
720        if first {
721            c.cur = 0;
722            c.next = want.start / block * block;
723            c.end = want.end.div_ceil(block) * block;
724        }
725        Ok(())
726    }
727
728    /// `bytes` (an export's slot object) as one chunk at stream offset
729    /// `offset`, in a buffer of the stream's: lent like a read's.
730    ///
731    /// # Errors
732    ///
733    /// `NotWritten(TooManyInFlight)` with every one of its `lend` chunks
734    /// held (or no buffer free); `NotWritten(TooLarge)` for bytes longer
735    /// than a buffer; `NotWritten(Interrupted)` once the stream has ended.
736    pub(crate) fn lend_bytes(
737        &self,
738        r: &LiveRegion,
739        offset: u64,
740        bytes: &[u8],
741    ) -> Result<Chunk, Error> {
742        let c = self.core.borrow_mut();
743        let ctx = || ctx_range(r, offset, offset + bytes.len() as u64);
744        if c.ended {
745            return Err(not_written(NotWrittenCause::Interrupted, ctx()));
746        }
747        let mut home = lock(&c.home);
748        if home.lent >= c.lend {
749            return Err(not_written(NotWrittenCause::TooManyInFlight, ctx()));
750        }
751        let Some(mut buf) = home.bufs.pop() else {
752            return Err(not_written(NotWrittenCause::TooManyInFlight, ctx()));
753        };
754        if !buf.set_len(bytes.len()) {
755            home.bufs.push(buf);
756            return Err(not_written(NotWrittenCause::TooLarge, ctx()));
757        }
758        buf.as_mut_capacity()[..bytes.len()].copy_from_slice(bytes);
759        home.lent += 1;
760        drop(home);
761        Ok(Chunk {
762            buf: Some(buf),
763            at: 0,
764            len: bytes.len(),
765            offset,
766            range: 0,
767            home: Arc::clone(&c.home),
768        })
769    }
770
771    /// The next item, as [`Iterator::next`], through a shared handle (an
772    /// export's, which adds parts between items).
773    pub(crate) fn pull(&self) -> Option<Result<Chunk, Error>> {
774        self.core.borrow_mut().next_item()
775    }
776
777    /// A part of the call (an export's sink) joined the stream's record:
778    /// the stream leaves it one of the call's unpaced pieces
779    /// ([`Pace::second`]), so its read-ahead never starves the sink.
780    pub(crate) fn make_second(&self) {
781        let mut c = self.core.borrow_mut();
782        c.pace = c.pace.as_ref().map(Pace::second);
783    }
784
785    /// The error of a collection for another part of the call that lost
786    /// the stream's reads (sticky), as a read error of the part then read.
787    pub(crate) fn failure(&self) -> Option<Error> {
788        let c = self.core.try_borrow().ok()?;
789        c.collect_failed.map(|(raw, _)| c.read_error(raw))
790    }
791
792    /// The chunks its caller may hold at once.
793    pub(crate) fn lend(&self) -> usize {
794        self.core.borrow().lend
795    }
796
797    /// The stream's pacing (a `Background` stream's call).
798    #[cfg(all(test, not(loom)))]
799    pub(crate) fn pace(&self) -> Option<Pace> {
800        self.core.borrow().pace.clone()
801    }
802}
803
804impl<P: Platform> Collect for RefCell<Core<P>>
805where
806    P::Queue: Send,
807{
808    fn collect_one(&self) -> Result<bool, OsError> {
809        // Borrowed: this stream is the part asking, which collected its
810        // own reads first.
811        match self.try_borrow_mut() {
812            Ok(mut c) => c.collect_for_another(false),
813            Err(_) => Ok(false),
814        }
815    }
816
817    fn collect_all(&self) -> Result<(), OsError> {
818        match self.try_borrow_mut() {
819            Ok(mut c) => c.collect_for_another(true).map(|_| ()),
820            Err(_) => Ok(()),
821        }
822    }
823
824    fn failed(&self) -> Option<(OsError, u64)> {
825        self.try_borrow().ok()?.collect_failed
826    }
827}
828
829impl<P: Platform> Core<P>
830where
831    P::Queue: Send,
832{
833    /// Collects reads in flight for another part of the call (one at
834    /// least, or all): whether there were any. A failed wait is recorded
835    /// (sticky: the next `next()` ends the stream with it) and every read's
836    /// charge dropped (the reads stay with the queue, which is never waited
837    /// on again).
838    fn collect_for_another(&mut self, all: bool) -> Result<bool, OsError> {
839        // Ended: it holds nothing (a failure already reported is its own).
840        if self.finished {
841            return Ok(false);
842        }
843        if let Some((raw, _)) = self.collect_failed {
844            return Err(raw);
845        }
846        if self.in_flight == 0 {
847            return Ok(false);
848        }
849        let waited = if all {
850            self.collect_all()
851        } else {
852            self.wait_one()
853        };
854        match waited {
855            Ok(()) => Ok(true),
856            Err(raw) => {
857                let lost = self
858                    .ring
859                    .iter()
860                    .flatten()
861                    .filter(|r| r.done.is_none())
862                    .map(|r| r.start)
863                    .min()
864                    .unwrap_or(self.next);
865                self.collect_failed = Some((raw, lost));
866                self.wait_failed = Some(raw);
867                for r in self.ring.iter_mut().flatten() {
868                    r.charge = None;
869                }
870                Err(raw)
871            }
872        }
873    }
874
875    /// Part `i`'s range as an error's context.
876    fn ctx_of(&self, i: usize) -> ErrorContext {
877        match self.parts.get(i) {
878            Some(p) => ctx_range(&p.r, p.want.start, p.want.end),
879            None => ErrorContext::default(),
880        }
881    }
882
883    /// The current part's.
884    fn ctx(&self) -> ErrorContext {
885        self.ctx_of(self.cur)
886    }
887
888    fn read_error_of(&self, i: usize, raw: OsError) -> Error {
889        Error::Io {
890            op: Op::Read,
891            raw,
892            ctx: self.ctx_of(i),
893        }
894    }
895
896    fn read_error(&self, raw: OsError) -> Error {
897        self.read_error_of(self.cur, raw)
898    }
899
900    /// Part `i`'s region's gate, checked without a pass (`Gate::live`).
901    fn live_of(&self, i: usize) -> Result<(), Error> {
902        match self.parts.get(i) {
903            Some(p) => p.r.gate.live().map_err(|why| retired_error(why, &p.r)),
904            None => Err(not_written(NotWrittenCause::Interrupted, self.ctx())),
905        }
906    }
907
908    /// Whether data is left to read: moves to the next part once the
909    /// current one is all submitted.
910    fn more(&mut self) -> bool {
911        while self.next >= self.end {
912            if self.cur + 1 >= self.parts.len() {
913                return false;
914            }
915            self.cur += 1;
916            let block = self.store.block();
917            let p = &self.parts[self.cur];
918            self.next = p.want.start / block * block;
919            self.end = p.want.end.div_ceil(block) * block;
920        }
921        true
922    }
923
924    /// A bisected piece's run waits to be copied out of its buffer: no read
925    /// is submitted meanwhile, so a free buffer stays for it.
926    fn copying(&self) -> bool {
927        self.pending
928            .iter()
929            .any(|i| matches!(i, Item::Run { last: false, .. }))
930    }
931
932    fn depth(&self) -> usize {
933        self.ring.len()
934    }
935
936    /// Submits reads while the ring has room, a buffer is free and data is
937    /// left, each read cut where its extent ends.
938    fn refill(&mut self) -> Result<(), Error> {
939        let depth = self.depth() as u64;
940        while !self.copying() && self.submitted - self.delivered < depth && self.more() {
941            let part = self.cur;
942            self.live_of(part)?;
943            test_hook::step(Step::StreamSubmit);
944            let Some(mut buf) = lock(&self.home).bufs.pop() else {
945                break;
946            };
947            let next = self.next;
948            // Never across a run's end (one device operation each).
949            let len = (self.end - next)
950                .min(self.req as u64)
951                .min(self.parts[part].run_left(next)) as usize;
952            // A part whose runs end before its bytes do (a backstop: never
953            // a read of nothing, which would never end).
954            if len == 0 {
955                lock(&self.home).bufs.push(buf);
956                return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
957            }
958            // A background read is admitted before it is submitted; refused
959            // with reads of its own in flight, the stream collects them.
960            let charge = match &self.pace {
961                None => None,
962                Some(p) => match p.try_charge() {
963                    Some(ch) => Some(ch),
964                    None if self.in_flight > 0 => {
965                        lock(&self.home).bufs.push(buf);
966                        break;
967                    }
968                    None => match p.wait_charge() {
969                        Some(ch) => Some(ch),
970                        None => {
971                            lock(&self.home).bufs.push(buf);
972                            return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
973                        }
974                    },
975                },
976            };
977            // `len ≤ req ≤ capacity`.
978            let _fits = buf.set_len(len);
979            let op = IoOp::Read {
980                file: &self.store.inner.file,
981                offset: self.parts[part].locate(next, len as u64),
982                buf,
983            };
984            let Some(lane) = self.lane.as_mut() else {
985                return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
986            };
987            match lane.submit(op, self.submitted) {
988                Ok(()) => {
989                    let slot = (self.submitted % depth) as usize;
990                    self.ring[slot] = Some(Read {
991                        part,
992                        start: next,
993                        len,
994                        done: None,
995                        charge,
996                    });
997                    self.submitted += 1;
998                    self.next += len as u64;
999                    self.in_flight += 1;
1000                }
1001                Err(rejected) => {
1002                    if let Some(b) = rejected.buf {
1003                        lock(&self.home).bufs.push(b);
1004                    }
1005                    match rejected.raw {
1006                        None if self.in_flight > 0 => break,
1007                        None => {
1008                            return Err(not_written(NotWrittenCause::QueueFull, self.ctx()));
1009                        }
1010                        Some(raw) => return Err(self.read_error(raw)),
1011                    }
1012                }
1013            }
1014        }
1015        Ok(())
1016    }
1017
1018    /// Files every completion the lane holds under its read (its charge
1019    /// released).
1020    fn file(&mut self) {
1021        let depth = self.depth() as u64;
1022        let Some(lane) = self.lane.as_mut() else {
1023            return;
1024        };
1025        for comp in lane.done.drain() {
1026            self.in_flight = self.in_flight.saturating_sub(1);
1027            if let Some(p) = self.ring[(comp.tag % depth) as usize].as_mut() {
1028                p.done = Some((comp.result, comp.buf));
1029                p.charge = None;
1030            }
1031        }
1032    }
1033
1034    /// Collects what has completed, without waiting.
1035    fn reap(&mut self) {
1036        if let Some(lane) = self.lane.as_mut() {
1037            let _arrived = lane.reap();
1038        }
1039        self.file();
1040    }
1041
1042    /// Waits for at least one read in flight.
1043    fn wait_one(&mut self) -> Result<(), OsError> {
1044        let waited = match self.lane.as_mut() {
1045            Some(lane) => lane.wait(1).map(|_| ()),
1046            None => Ok(()),
1047        };
1048        self.file();
1049        waited
1050    }
1051
1052    /// Waits for every read in flight.
1053    fn collect_all(&mut self) -> Result<(), OsError> {
1054        while self.in_flight > 0 {
1055            self.wait_one()?;
1056        }
1057        Ok(())
1058    }
1059
1060    /// A chunk of part `part`, `len` bytes of `buf` from `at`, region
1061    /// offset `offset`: one more lent.
1062    fn lend_chunk(&mut self, part: usize, buf: IoBuf, at: usize, len: usize, offset: u64) -> Chunk {
1063        lock(&self.home).lent += 1;
1064        Chunk {
1065            buf: Some(buf),
1066            at,
1067            len,
1068            offset,
1069            range: self.parts.get(part).map_or(0, |p| p.index),
1070            home: Arc::clone(&self.home),
1071        }
1072    }
1073
1074    /// A range of part `part` that could not be read, as its item.
1075    fn bad(&self, part: usize, start: u64, end: u64) -> Error {
1076        Error::Corruption {
1077            kind: CorruptionKind::MediaError,
1078            ctx: match self.parts.get(part) {
1079                Some(p) => ctx_range(&p.r, start, end),
1080                None => ErrorContext::default(),
1081            },
1082        }
1083    }
1084
1085    /// Queues `[start, end)` of part `part` as unreadable, counted once as
1086    /// a media error (after its check).
1087    fn queue_bad(&mut self, part: usize, start: u64, end: u64) {
1088        self.store.inner.observer.add(Counter::MediaErrors, 1);
1089        self.pending.push_back(Item::Bad { part, start, end });
1090    }
1091
1092    /// `[start, end)` clamped to part `part`'s bytes asked for (empty:
1093    /// `None`).
1094    fn clamp(&self, part: usize, start: u64, end: u64) -> Option<(u64, u64)> {
1095        let want = &self.parts.get(part)?.want;
1096        let (s, e) = (start.max(want.start), end.min(want.end));
1097        (s < e).then_some((s, e))
1098    }
1099
1100    /// The next item: refill, collect, deliver. `Ok(None)`: the stream's
1101    /// end; `Err`: an error that ends it.
1102    fn advance(&mut self) -> Result<Next, Error> {
1103        loop {
1104            self.refill()?;
1105            self.reap();
1106            // Decided before delivering anything (the refill above still
1107            // keeps reads in flight): a refused call delivers, counts and
1108            // classifies nothing.
1109            if lock(&self.home).lent >= self.lend {
1110                return Ok(Some(Err(not_written(
1111                    NotWrittenCause::TooManyInFlight,
1112                    self.ctx(),
1113                ))));
1114            }
1115            if let Some(item) = self.pending.pop_front() {
1116                return Ok(Some(self.deliver_item(item)));
1117            }
1118            if self.delivered == self.submitted {
1119                if !self.more() {
1120                    return Ok(None);
1121                }
1122                // Nothing in flight and nothing submitted: nothing ever
1123                // will be (a backstop: the stream fails, never spins).
1124                return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
1125            }
1126            let depth = self.depth() as u64;
1127            let slot = (self.delivered % depth) as usize;
1128            let waiting = self.ring[slot].as_ref().map_or(self.cur, |p| p.part);
1129            while self.ring[slot].as_ref().is_some_and(|p| p.done.is_none()) {
1130                if let Err(raw) = self.wait_one() {
1131                    return Err(self.read_error_of(waiting, raw));
1132                }
1133            }
1134            let Some(Read {
1135                part,
1136                start,
1137                len,
1138                done: Some((result, buf)),
1139                ..
1140            }) = self.ring[slot].take()
1141            else {
1142                return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
1143            };
1144            self.delivered += 1;
1145            test_hook::step(Step::StreamCompleted);
1146            // Before anything of the read is delivered, counted or
1147            // classified (an error's too): its own part's gate.
1148            if let Err(e) = self.live_of(part) {
1149                if let Some(b) = buf {
1150                    lock(&self.home).bufs.push(b);
1151                }
1152                return Err(e);
1153            }
1154            match (result, buf) {
1155                (Ok(n), Some(b)) => {
1156                    let n = n.min(len);
1157                    let obs = &self.store.inner.observer;
1158                    obs.add(Counter::Reads, 1);
1159                    obs.add(Counter::ReadBytes, n as u64);
1160                    // A short read: the rest could not be read.
1161                    if let Some((s, e)) = self.clamp(part, start + n as u64, start + len as u64) {
1162                        self.queue_bad(part, s, e);
1163                    }
1164                    match self.clamp(part, start, start + n as u64) {
1165                        Some((s, e)) => {
1166                            let at = (s - start) as usize;
1167                            let chunk = self.lend_chunk(part, b, at, (e - s) as usize, s);
1168                            return Ok(Some(Ok(chunk)));
1169                        }
1170                        None => lock(&self.home).bufs.push(b),
1171                    }
1172                }
1173                (Err(raw), Some(b)) if is_media_error(raw) => {
1174                    self.bisect(part, start, len as u64, b)?;
1175                }
1176                (Err(raw), b) => {
1177                    if let Some(b) = b {
1178                        lock(&self.home).bufs.push(b);
1179                    }
1180                    return Err(self.read_error_of(part, raw));
1181                }
1182                // No buffer came back (a backend breaking platform contract
1183                // 1): the stream stops, never reporting the range as bad.
1184                (Ok(_), None) => {
1185                    return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
1186                }
1187            }
1188        }
1189    }
1190
1191    /// A bisected piece's next item.
1192    fn deliver_item(&mut self, item: Item) -> Result<Chunk, Error> {
1193        match item {
1194            Item::Bad { part, start, end } => Err(self.bad(part, start, end)),
1195            Item::Run {
1196                part,
1197                start,
1198                at,
1199                len,
1200                last: true,
1201            } => match self.piece.take() {
1202                Some(b) => Ok(self.lend_chunk(part, b, at, len, start)),
1203                None => Err(not_written(NotWrittenCause::Interrupted, self.ctx())),
1204            },
1205            Item::Run {
1206                part,
1207                start,
1208                at,
1209                len,
1210                ..
1211            } => {
1212                let free = lock(&self.home).bufs.pop();
1213                let (Some(mut buf), Some(piece)) = (free, self.piece.as_ref()) else {
1214                    // No free buffer while fewer than `lend` are lent cannot
1215                    // happen (no read is submitted while a run waits); kept
1216                    // for its turn.
1217                    self.pending.push_front(item);
1218                    return Err(not_written(NotWrittenCause::TooManyInFlight, self.ctx()));
1219                };
1220                let _fits = buf.set_len(len);
1221                buf.as_mut_capacity()[..len].copy_from_slice(&piece.as_slice()[at..at + len]);
1222                Ok(self.lend_chunk(part, buf, 0, len, start))
1223            }
1224        }
1225    }
1226
1227    /// Bisects the piece `[start, start + len)`, whose read failed with a
1228    /// media error, on the stream's idle queue; its items are queued only
1229    /// after the gate holds once all its reads have completed.
1230    fn bisect(&mut self, part: usize, start: u64, len: u64, mut piece: IoBuf) -> Result<(), Error> {
1231        // An idle queue: every read in flight collected first (delivered
1232        // later, in order, after this piece).
1233        if let Err(raw) = self.collect_all() {
1234            lock(&self.home).bufs.push(piece);
1235            return Err(self.read_error_of(part, raw));
1236        }
1237        let mut scratch = self.scratch.take();
1238        let mut found: Vec<(u64, u64, bool)> = Vec::new();
1239        let halves = self.halves(
1240            part,
1241            start,
1242            start,
1243            len,
1244            &mut piece,
1245            &mut scratch,
1246            &mut found,
1247        );
1248        self.scratch = scratch;
1249        if let Err(e) = halves {
1250            self.bisected = (0, 0);
1251            lock(&self.home).bufs.push(piece);
1252            return Err(e);
1253        }
1254        test_hook::step(Step::StreamBisected);
1255        let (reads, bytes) = std::mem::take(&mut self.bisected);
1256        if let Err(e) = self.live_of(part) {
1257            lock(&self.home).bufs.push(piece);
1258            return Err(e);
1259        }
1260        let obs = &self.store.inner.observer;
1261        obs.add(Counter::Reads, reads);
1262        obs.add(Counter::ReadBytes, bytes);
1263        // Merged (good with good, bad with bad), clamped to the bytes asked
1264        // for.
1265        let mut merged: Vec<(u64, u64, bool)> = Vec::new();
1266        for (s, e, good) in found {
1267            match merged.last_mut() {
1268                Some((_, le, lg)) if *lg == good && *le == s => *le = e,
1269                _ => merged.push((s, e, good)),
1270            }
1271        }
1272        let items: Vec<Item> = merged
1273            .into_iter()
1274            .filter_map(|(s, e, good)| {
1275                let (s, e) = self.clamp(part, s, e)?;
1276                Some(if good {
1277                    Item::Run {
1278                        part,
1279                        start: s,
1280                        at: (s - start) as usize,
1281                        len: (e - s) as usize,
1282                        last: false,
1283                    }
1284                } else {
1285                    Item::Bad {
1286                        part,
1287                        start: s,
1288                        end: e,
1289                    }
1290                })
1291            })
1292            .collect();
1293        let last_run = items.iter().rposition(|i| matches!(i, Item::Run { .. }));
1294        for (i, item) in items.into_iter().enumerate() {
1295            match item {
1296                Item::Run {
1297                    part,
1298                    start,
1299                    at,
1300                    len,
1301                    ..
1302                } => self.pending.push_back(Item::Run {
1303                    part,
1304                    start,
1305                    at,
1306                    len,
1307                    last: Some(i) == last_run,
1308                }),
1309                Item::Bad { part, start, end } => self.queue_bad(part, start, end),
1310            }
1311        }
1312        match last_run {
1313            // Lent with its last run.
1314            Some(_) => self.piece = Some(piece),
1315            // No good run: its buffer back.
1316            None => lock(&self.home).bufs.push(piece),
1317        }
1318        Ok(())
1319    }
1320
1321    /// Reads `[at, at + len)` again in halves into `scratch`, each good
1322    /// result copied to its place in `piece` (which starts at `base`),
1323    /// down to one sector: `found` gets `(start, end, good)` in order.
1324    /// `scratch` is left `None` if the queue kept it (lost).
1325    #[allow(clippy::too_many_arguments)]
1326    fn halves(
1327        &mut self,
1328        part: usize,
1329        base: u64,
1330        at: u64,
1331        len: u64,
1332        piece: &mut IoBuf,
1333        scratch: &mut Option<IoBuf>,
1334        found: &mut Vec<(u64, u64, bool)>,
1335    ) -> Result<(), Error> {
1336        let sector = self.store.sector();
1337        let block = self.store.block();
1338        let Some(mut buf) = scratch.take() else {
1339            return Err(not_written(NotWrittenCause::Interrupted, self.ctx_of(part)));
1340        };
1341        // `len ≤ req ≤ capacity`.
1342        let _fits = buf.set_len(len as usize);
1343        let admitted = match &self.pace {
1344            Some(p) => Some(p.admit()?),
1345            None => None,
1346        };
1347        let op = IoOp::Read {
1348            file: &self.store.inner.file,
1349            offset: self.parts[part].locate(at, len),
1350            buf,
1351        };
1352        let Some(lane) = self.lane.as_mut() else {
1353            return Err(not_written(NotWrittenCause::Interrupted, self.ctx_of(part)));
1354        };
1355        let done = run(lane, op);
1356        // Completed (or abandoned): its charge goes with it.
1357        drop(admitted);
1358        let (result, back) = match done {
1359            Ok(Ran::Done { buf, result }) => (result, buf),
1360            Ok(Ran::Abandoned(raw)) => return Err(self.read_error_of(part, raw)),
1361            Err((buf, Some(raw))) => {
1362                *scratch = buf;
1363                return Err(self.read_error_of(part, raw));
1364            }
1365            Err((buf, None)) => {
1366                *scratch = buf;
1367                return Err(not_written(NotWrittenCause::QueueFull, self.ctx_of(part)));
1368            }
1369        };
1370        let Some(b) = back else {
1371            return Err(match result {
1372                Err(raw) => self.read_error_of(part, raw),
1373                Ok(_) => not_written(NotWrittenCause::Interrupted, self.ctx_of(part)),
1374            });
1375        };
1376        match result {
1377            Ok(n) => {
1378                let n = (n as u64).min(len);
1379                let place = (at - base) as usize;
1380                piece.as_mut_capacity()[place..place + n as usize]
1381                    .copy_from_slice(&b.as_slice()[..n as usize]);
1382                *scratch = Some(b);
1383                // Counted once the bisection's check holds.
1384                self.bisected.0 += 1;
1385                self.bisected.1 += n;
1386                if n > 0 {
1387                    found.push((at, at + n, true));
1388                }
1389                if n < len {
1390                    found.push((at + n, at + len, false));
1391                }
1392                Ok(())
1393            }
1394            Err(raw) if is_media_error(raw) => {
1395                *scratch = Some(b);
1396                if len <= sector {
1397                    found.push((at, at + len, false));
1398                    return Ok(());
1399                }
1400                // Whole blocks while the piece spans several; sectors only
1401                // inside one block (as the scan's bisection).
1402                let unit = if len > block { block } else { sector };
1403                let half = (len / unit).div_ceil(2) * unit;
1404                self.halves(part, base, at, half, piece, scratch, found)?;
1405                self.halves(part, base, at + half, len - half, piece, scratch, found)
1406            }
1407            // Below the store block the device refused the smaller read
1408            // (alignment): it lies in a block that failed on the media.
1409            Err(_) if len < block => {
1410                *scratch = Some(b);
1411                found.push((at, at + len, false));
1412                Ok(())
1413            }
1414            Err(raw) => {
1415                *scratch = Some(b);
1416                Err(self.read_error_of(part, raw))
1417            }
1418        }
1419    }
1420
1421    /// Ends the stream: the reads in flight waited for (a failed wait kept
1422    /// for `cancel`; their buffers stay with the queue), every buffer not
1423    /// lent back to the pool, the list closed so chunks still held go to
1424    /// the pool as they drop.
1425    fn finish(&mut self) {
1426        if self.finished {
1427            return;
1428        }
1429        self.finished = true;
1430        self.ended = true;
1431        if self.wait_failed.is_none() {
1432            if let Err(raw) = self.collect_all() {
1433                self.wait_failed = Some(raw);
1434            }
1435        }
1436        // Their charges and buffers go with them.
1437        self.ring.iter_mut().for_each(|slot| *slot = None);
1438        self.pending.clear();
1439        self.piece = None;
1440        self.scratch = None;
1441        let bufs = lock(&self.home).close();
1442        drop(bufs);
1443    }
1444}
1445
1446impl<P: Platform> Iterator for ReadStream<P>
1447where
1448    P::Queue: Send,
1449{
1450    type Item = Result<Chunk, Error>;
1451
1452    fn next(&mut self) -> Option<Self::Item> {
1453        self.core.borrow_mut().next_item()
1454    }
1455}
1456
1457impl<P: Platform> Core<P>
1458where
1459    P::Queue: Send,
1460{
1461    /// `Iterator::next`, under the cell's one borrow.
1462    fn next_item(&mut self) -> Option<Result<Chunk, Error>> {
1463        if self.ended {
1464            return None;
1465        }
1466        // A collection for another part failed: the reads it lost end the
1467        // stream.
1468        if let Some((raw, _)) = self.collect_failed {
1469            self.finish();
1470            return Some(Err(self.read_error(raw)));
1471        }
1472        match self.advance() {
1473            Ok(Some(item)) => Some(item),
1474            Ok(None) => {
1475                self.finish();
1476                None
1477            }
1478            Err(e) => {
1479                self.finish();
1480                Some(Err(e))
1481            }
1482        }
1483    }
1484}
1485
1486impl<P: Platform> Drop for Core<P>
1487where
1488    P::Queue: Send,
1489{
1490    fn drop(&mut self) {
1491        self.finish();
1492        if let Some(id) = self.mark.take() {
1493            self.store.unregister_submitter(id);
1494        }
1495    }
1496}
1497
1498#[cfg(all(test, not(loom)))]
1499mod tests {
1500    use super::*;
1501
1502    #[test]
1503    fn test_a_chunk_is_send() {
1504        fn send<T: Send>() {}
1505        send::<Chunk>();
1506    }
1507
1508    #[test]
1509    fn test_a_closed_list_hands_buffers_back() {
1510        let mut l = FreeList::new(vec![1u32, 2]);
1511        l.lent = 1;
1512        assert_eq!(l.give_back(3), None);
1513        assert_eq!(l.close(), vec![1, 2, 3]);
1514        l.lent = 1;
1515        assert_eq!(l.give_back(4), Some(4));
1516        assert_eq!(l.lent, 0);
1517    }
1518}
1519
1520/// A stream as one part of a call (V6): another part on its thread
1521/// collects its reads, and its own admission skips it while it asks.
1522#[cfg(all(test, not(loom)))]
1523mod collect_tests {
1524    #![allow(clippy::unwrap_used, clippy::expect_used, unused_results)]
1525
1526    use std::cell::RefCell;
1527    use std::path::Path;
1528    use std::rc::Rc;
1529
1530    use store_io_sim::{SimConfig, SimPlatform};
1531
1532    use super::*;
1533    use crate::{AppendRegion, StoreOptions};
1534
1535    const LEN: usize = 4 << 20;
1536
1537    /// A store whose commits are quiet and whose calls keep at most four
1538    /// pieces in flight unpaced, and a region of `LEN` stamped bytes,
1539    /// durable.
1540    fn setup(
1541        seed: u64,
1542    ) -> (
1543        SimPlatform,
1544        Store<SimPlatform>,
1545        AppendRegion<SimPlatform>,
1546        Vec<u8>,
1547    ) {
1548        let p = SimPlatform::new(SimConfig::volatile(seed));
1549        let opts = StoreOptions {
1550            background_unpaced: 4,
1551            ..StoreOptions::default()
1552        };
1553        let s = Store::create(p.clone(), Path::new("/s"), opts).unwrap();
1554        let log = s.provision_append_region("log", LEN as u64).unwrap();
1555        let data: Vec<u8> = (0..LEN).map(|i| (i % 251) as u8).collect();
1556        let t = log.append(&data).unwrap();
1557        let _receipt = log.sync_through(&t).unwrap();
1558        p.with_world(|w| w.advance_clock_ns(10_000_000_000));
1559        assert!(!s.qos().active);
1560        (p, s, log, data)
1561    }
1562
1563    /// The stream's next chunk, appended to `out` (checked in order).
1564    fn take(stream: &mut ReadStream<SimPlatform>, out: &mut Vec<u8>) {
1565        let c = stream.next().unwrap().unwrap();
1566        assert_eq!(c.offset() as usize, out.len());
1567        out.extend_from_slice(c.bytes());
1568    }
1569
1570    /// Two chunks read, the second's refill left in flight: the stream
1571    /// then holds the call's four unpaced pieces.
1572    fn holding_the_cap(p: &SimPlatform, stream: &mut ReadStream<SimPlatform>, out: &mut Vec<u8>) {
1573        take(stream, out);
1574        p.with_world(|w| w.hold_reaps(1));
1575        take(stream, out);
1576        assert_eq!(stream.pace().unwrap().held_here(), 4, "the call's cap");
1577    }
1578
1579    /// Another part of a stream's call, refused because the stream's reads
1580    /// hold the call's cap, collects them on this thread and is admitted;
1581    /// the stream then delivers every byte in order.
1582    #[test]
1583    fn test_another_part_collects_a_stream_s_reads() {
1584        let (p, _s, log, data) = setup(70);
1585        let mut stream = log.read_stream(0..LEN as u64).unwrap();
1586        let mut out = Vec::new();
1587        holding_the_cap(&p, &mut stream, &mut out);
1588        let pace = stream.pace().unwrap();
1589        let piece = pace.admit().unwrap();
1590        assert_eq!(pace.held_here(), 1, "the stream's reads collected");
1591        drop(piece);
1592        while out.len() < LEN {
1593            take(&mut stream, &mut out);
1594        }
1595        assert!(stream.next().is_none());
1596        assert!(out == data);
1597    }
1598
1599    /// Paced, another part refused collects all of the stream's reads.
1600    #[test]
1601    fn test_a_paced_part_collects_all_of_a_stream_s_reads() {
1602        let (p, s, log, data) = setup(73);
1603        let mut stream = log.read_stream(0..LEN as u64).unwrap();
1604        let mut out = Vec::new();
1605        holding_the_cap(&p, &mut stream, &mut out);
1606        let hold = s.commit_hold();
1607        let pace = stream.pace().unwrap();
1608        let piece = pace.admit().unwrap();
1609        assert_eq!(pace.held_here(), 1);
1610        drop(piece);
1611        drop(hold);
1612        p.with_world(|w| w.advance_clock_ns(10_000_000_000));
1613        while out.len() < LEN {
1614            take(&mut stream, &mut out);
1615        }
1616        assert!(out == data);
1617    }
1618
1619    /// A stream with nothing in flight is never waited on for another part.
1620    #[test]
1621    fn test_a_stream_with_nothing_in_flight_is_not_waited_on() {
1622        let (p, _s, log, _data) = setup(74);
1623        let mut stream = log.read_stream(0..LEN as u64).unwrap();
1624        let mut out = Vec::new();
1625        take(&mut stream, &mut out);
1626        let pace = stream.pace().unwrap();
1627        assert_eq!(pace.held_here(), 0, "the sim's wait completed every read");
1628        let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
1629        let other = Rc::new(Other(RefCell::new(held)));
1630        let weak: Weak<Other> = Rc::downgrade(&other);
1631        let _reg = pace.register(weak);
1632        let waits = p.with_world(|w| w.waits());
1633        let _piece = pace.admit().unwrap();
1634        assert_eq!(p.with_world(|w| w.waits()), waits);
1635        assert_eq!(other.0.borrow().len(), 3);
1636    }
1637
1638    /// Another part of a call holding its pieces.
1639    struct Other(RefCell<Vec<Held>>);
1640
1641    impl Collect for Other {
1642        fn collect_one(&self) -> Result<bool, OsError> {
1643            Ok(self.0.borrow_mut().pop().is_some())
1644        }
1645
1646        fn collect_all(&self) -> Result<(), OsError> {
1647            self.0.borrow_mut().clear();
1648            Ok(())
1649        }
1650
1651        fn failed(&self) -> Option<(OsError, u64)> {
1652            None
1653        }
1654    }
1655
1656    /// A stream refused because another part on its thread holds the
1657    /// call's cap collects that part's pieces, skipping its own collector
1658    /// (its cell is borrowed: it is the part asking), and reads on.
1659    #[test]
1660    fn test_a_stream_asking_skips_itself_and_collects_the_other_part() {
1661        let (_p, _s, log, data) = setup(71);
1662        let mut stream = log.read_stream(0..LEN as u64).unwrap();
1663        let pace = stream.pace().unwrap();
1664        let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
1665        let other = Rc::new(Other(RefCell::new(held)));
1666        let weak: Weak<Other> = Rc::downgrade(&other);
1667        let _reg = pace.register(weak);
1668        let mut out = Vec::new();
1669        while out.len() < LEN {
1670            take(&mut stream, &mut out);
1671        }
1672        assert!(stream.next().is_none());
1673        assert!(out == data);
1674        assert!(
1675            other.0.borrow().len() < 4,
1676            "the other part's pieces collected"
1677        );
1678    }
1679
1680    /// Paced, a stream asking while another part holds the call's pieces
1681    /// collects them all, skipping its own collector, and reads on.
1682    #[test]
1683    fn test_a_paced_stream_asking_skips_itself_and_collects_the_other_part() {
1684        let (_p, s, log, data) = setup(75);
1685        let mut stream = log.read_stream(0..LEN as u64).unwrap();
1686        let pace = stream.pace().unwrap();
1687        let held = (0..2).map(|_| pace.try_charge().unwrap()).collect();
1688        let other = Rc::new(Other(RefCell::new(held)));
1689        let weak: Weak<Other> = Rc::downgrade(&other);
1690        let _reg = pace.register(weak);
1691        let _hold = s.commit_hold();
1692        let mut out = Vec::new();
1693        while out.len() < LEN {
1694            take(&mut stream, &mut out);
1695        }
1696        assert!(out == data);
1697        assert!(other.0.borrow().is_empty());
1698    }
1699
1700    /// A collection for another part whose wait fails: the admission ends,
1701    /// the stream's charges are dropped (the reads stay with its queue,
1702    /// never waited on again), the failure is recorded, and the stream's
1703    /// next call ends it with the error.
1704    #[test]
1705    fn test_a_failed_collection_ends_the_stream_with_its_error() {
1706        let (p, _s, log, _data) = setup(72);
1707        let mut stream = log.read_stream(0..LEN as u64).unwrap();
1708        let mut out = Vec::new();
1709        holding_the_cap(&p, &mut stream, &mut out);
1710        let pace = stream.pace().unwrap();
1711        p.with_world(|w| {
1712            let next = w.waits() + 1;
1713            w.faults_mut().fail_wait = Some((next, 5));
1714        });
1715        assert!(pace.admit().is_err());
1716        assert_eq!(pace.held_here(), 0, "its charges dropped");
1717        // Two chunks delivered, two more read: the oldest lost is the fifth.
1718        let lost = stream.core.failed().map(|(_, at)| at);
1719        assert_eq!(lost, Some(4 * (out.len() as u64 / 2)));
1720        let waits = p.with_world(|w| w.waits());
1721        // Sticky: another admission fails without waiting again.
1722        let again = pace.try_charge().unwrap();
1723        let more: Vec<_> = (0..3).map(|_| pace.try_charge().unwrap()).collect();
1724        assert!(pace.admit().is_err());
1725        assert_eq!(p.with_world(|w| w.waits()), waits);
1726        drop((again, more));
1727        assert!(matches!(stream.next(), Some(Err(Error::Io { .. }))));
1728        assert!(stream.next().is_none());
1729        // Ended, it holds nothing: never a failure for the call's others.
1730        let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
1731        let other = Rc::new(Other(RefCell::new(held)));
1732        let weak: Weak<Other> = Rc::downgrade(&other);
1733        let _reg = pace.register(weak);
1734        let _piece = pace.admit().unwrap();
1735        drop(stream);
1736        assert_eq!(p.with_world(|w| w.waits()), waits, "never waited on again");
1737    }
1738}
1739
1740#[cfg(all(test, loom))]
1741mod loom_tests {
1742    use super::*;
1743    use loom::thread;
1744
1745    /// A chunk's buffer given back while the stream closes its list: every
1746    /// buffer ends either on the drained list or handed back, never on a
1747    /// list already drained.
1748    #[test]
1749    fn loom_a_give_back_against_a_close_strands_nothing() {
1750        loom::model(|| {
1751            let home = Arc::new(Mutex::new(FreeList::new(vec![1u32])));
1752            lock(&home).lent = 1;
1753            let h = Arc::clone(&home);
1754            let t = thread::spawn(move || lock(&h).give_back(2));
1755            let drained = lock(&home).close();
1756            let back = t.join().unwrap_or(None);
1757            // Nothing left on the closed list (it would be stranded), and
1758            // both buffers accounted for.
1759            assert!(
1760                lock(&home).bufs.is_empty(),
1761                "a buffer pushed after the close"
1762            );
1763            assert_eq!(drained.len() + usize::from(back.is_some()), 2);
1764        });
1765    }
1766}