1use std::cell::RefCell;
32use std::collections::VecDeque;
33use std::marker::PhantomData;
34use std::ops::Range;
35use std::rc::{Rc, Weak};
36use std::sync::{Arc, PoisonError};
37
38#[cfg(loom)]
39use loom::sync::{Mutex, MutexGuard};
40#[cfg(not(loom))]
41use std::sync::{Mutex, MutexGuard};
42
43use store_io_core::IoClass;
44use store_io_core::errno::is_media_error;
45use store_io_core::error::{CorruptionKind, Error, ErrorContext, NotWrittenCause, Op, OsError};
46use store_io_core::observe::Counter;
47use store_io_platform::{IoBuf, IoOp, Platform, RawResult};
48
49use crate::exec::{Lane, LaneMark, Ran, owned_lane, run};
50use crate::io::{ctx_range, not_written};
51use crate::lifecycle::retired_error;
52use crate::pace::{Held, Pace, Registered};
53use crate::qos::Collect;
54use crate::store::{LiveRegion, Store};
55use crate::submitter::MAX_DEPTH;
56use crate::test_hook::{self, Step};
57
58const DEFAULT_DEPTH: usize = 8;
61
62#[derive(Clone, Copy, Debug, PartialEq, Eq)]
72#[non_exhaustive]
73pub struct StreamOptions {
74 depth: Option<usize>,
75 lend: Option<usize>,
76 class: IoClass,
77}
78
79impl Default for StreamOptions {
80 fn default() -> Self {
81 Self {
82 depth: None,
83 lend: None,
84 class: IoClass::Background,
85 }
86 }
87}
88
89impl StreamOptions {
90 #[must_use]
92 pub fn depth(mut self, depth: usize) -> Self {
93 self.depth = Some(depth);
94 self
95 }
96
97 #[must_use]
99 pub fn lend(mut self, lend: usize) -> Self {
100 self.lend = Some(lend);
101 self
102 }
103
104 #[must_use]
106 pub fn class(mut self, class: IoClass) -> Self {
107 self.class = class;
108 self
109 }
110}
111
112pub(crate) struct FreeList<B> {
118 closed: bool,
119 bufs: Vec<B>,
120 lent: usize,
121}
122
123impl<B> FreeList<B> {
124 pub(crate) fn new(bufs: Vec<B>) -> Self {
125 Self {
126 closed: false,
127 bufs,
128 lent: 0,
129 }
130 }
131
132 pub(crate) fn give_back(&mut self, b: B) -> Option<B> {
135 self.lent = self.lent.saturating_sub(1);
136 if self.closed {
137 Some(b)
138 } else {
139 self.bufs.push(b);
140 None
141 }
142 }
143
144 pub(crate) fn close(&mut self) -> Vec<B> {
146 self.closed = true;
147 std::mem::take(&mut self.bufs)
148 }
149}
150
151fn lock<B>(m: &Mutex<FreeList<B>>) -> MutexGuard<'_, FreeList<B>> {
152 m.lock().unwrap_or_else(PoisonError::into_inner)
153}
154
155pub struct Chunk {
159 buf: Option<IoBuf>,
160 at: usize,
161 len: usize,
162 offset: u64,
163 range: usize,
164 home: Arc<Mutex<FreeList<IoBuf>>>,
165}
166
167impl Chunk {
168 #[must_use]
172 pub fn offset(&self) -> u64 {
173 self.offset
174 }
175
176 #[must_use]
181 pub fn range(&self) -> usize {
182 self.range
183 }
184
185 #[must_use]
187 pub fn bytes(&self) -> &[u8] {
188 match &self.buf {
189 Some(b) => &b.as_slice()[self.at..self.at + self.len],
190 None => &[],
191 }
192 }
193}
194
195impl core::fmt::Debug for Chunk {
196 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
197 f.debug_struct("Chunk")
198 .field("offset", &self.offset)
199 .field("len", &self.len)
200 .field("range", &self.range)
201 .finish_non_exhaustive()
202 }
203}
204
205impl Drop for Chunk {
206 fn drop(&mut self) {
207 if let Some(b) = self.buf.take() {
208 let back = lock(&self.home).give_back(b);
211 drop(back);
212 }
213 }
214}
215
216struct Read {
218 part: usize,
220 start: u64,
221 len: usize,
222 done: Option<(RawResult<usize>, Option<IoBuf>)>,
223 charge: Option<Held>,
226}
227
228#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231enum Item {
232 Run {
235 part: usize,
236 start: u64,
237 at: usize,
238 len: usize,
239 last: bool,
240 },
241 Bad { part: usize, start: u64, end: u64 },
243}
244
245struct Part {
248 r: Arc<LiveRegion>,
249 want: Range<u64>,
250 index: usize,
251 loc: Locator,
252}
253
254pub(crate) enum Locator {
256 Map,
258 Ranges(Vec<(u64, u64)>),
264}
265
266impl Part {
267 fn run_left(&self, at: u64) -> u64 {
270 match &self.loc {
271 Locator::Map => self.r.map.run_left(at),
272 Locator::Ranges(ranges) => {
273 let mut base = 0u64;
274 for &(_, len) in ranges {
275 if at < base + len {
276 return base + len - at;
277 }
278 base += len;
279 }
280 0
281 }
282 }
283 }
284
285 fn locate(&self, at: u64, len: u64) -> u64 {
288 match &self.loc {
289 Locator::Map => self.r.map.locate(at, len),
290 Locator::Ranges(ranges) => {
291 let mut base = 0u64;
292 for &(file, n) in ranges {
293 if at < base + n {
294 return file + (at - base);
295 }
296 base += n;
297 }
298 u64::MAX
299 }
300 }
301 }
302}
303
304pub struct ReadStream<P: Platform>
332where
333 P::Queue: Send,
334{
335 core: Rc<RefCell<Core<P>>>,
340 _part: Option<Registered>,
341}
342
343struct Core<P: Platform>
347where
348 P::Queue: Send,
349{
350 lane: Option<Lane<P::Queue>>,
354 store: Store<P>,
355 parts: Vec<Part>,
357 cur: usize,
358 mark: Option<u64>,
360 ring: Vec<Option<Read>>,
361 submitted: u64,
362 delivered: u64,
363 in_flight: usize,
364 pending: VecDeque<Item>,
366 piece: Option<IoBuf>,
367 scratch: Option<IoBuf>,
368 home: Arc<Mutex<FreeList<IoBuf>>>,
369 lend: usize,
370 next: u64,
372 end: u64,
373 req: usize,
374 pace: Option<Pace>,
375 ended: bool,
376 finished: bool,
378 wait_failed: Option<OsError>,
379 bisected: (u64, u64),
381 collect_failed: Option<(OsError, u64)>,
384 _thread: PhantomData<*const ()>,
385}
386
387impl<P: Platform> core::fmt::Debug for ReadStream<P>
388where
389 P::Queue: Send,
390{
391 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
392 let mut d = f.debug_struct("ReadStream");
393 if let Ok(c) = self.core.try_borrow() {
394 let _d = d
395 .field("parts", &c.parts.len())
396 .field("cur", &c.cur)
397 .field("next", &c.next)
398 .field("ended", &c.ended);
399 }
400 d.finish_non_exhaustive()
401 }
402}
403
404type Next = Option<Result<Chunk, Error>>;
407
408pub struct StreamRange<'a, P: Platform> {
413 store: &'a Store<P>,
414 r: Arc<LiveRegion>,
415 want: Range<u64>,
416}
417
418impl<'a, P: Platform> StreamRange<'a, P> {
419 pub(crate) fn new(store: &'a Store<P>, r: &Arc<LiveRegion>, want: Range<u64>) -> Self {
420 Self {
421 store,
422 r: Arc::clone(r),
423 want,
424 }
425 }
426}
427
428impl<P: Platform> core::fmt::Debug for StreamRange<'_, P> {
429 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
430 f.debug_struct("StreamRange")
431 .field("region", &self.r.id)
432 .field("want", &self.want)
433 .finish_non_exhaustive()
434 }
435}
436
437impl<P: Platform> Store<P>
438where
439 P::Queue: Send,
440{
441 pub fn prefetch(
458 &self,
459 ranges: &[StreamRange<'_, P>],
460 opts: StreamOptions,
461 ) -> Result<ReadStream<P>, Error> {
462 for sr in ranges {
463 if !Arc::ptr_eq(&sr.store.inner, &self.inner) {
464 return Err(not_written(
465 NotWrittenCause::ForeignPosition,
466 ctx_range(&sr.r, sr.want.start, sr.want.end),
467 ));
468 }
469 sr.r.gate.live().map_err(|why| retired_error(why, &sr.r))?;
470 }
471 let parts = ranges
472 .iter()
473 .enumerate()
474 .map(|(i, sr)| (Arc::clone(&sr.r), sr.want.clone(), i))
475 .collect();
476 self.read_stream_of(parts, opts)
477 }
478}
479
480impl<P: Platform> Store<P>
481where
482 P::Queue: Send,
483{
484 pub(crate) fn read_stream_of(
488 &self,
489 parts: Vec<(Arc<LiveRegion>, Range<u64>, usize)>,
490 opts: StreamOptions,
491 ) -> Result<ReadStream<P>, Error> {
492 let pace = match opts.class {
494 IoClass::Read => None,
495 _ => self.inner.domain.new_pace(),
496 };
497 self.stream_of(parts, opts, pace, false)
498 }
499
500 pub(crate) fn export_stream(
506 &self,
507 depth: Option<usize>,
508 pace: Option<Pace>,
509 ) -> Result<ReadStream<P>, Error> {
510 let mut opts = StreamOptions::default();
511 if let Some(d) = depth {
512 opts = opts.depth(d);
513 }
514 self.stream_of(Vec::new(), opts, pace, true)
515 }
516
517 fn stream_of(
520 &self,
521 parts: Vec<(Arc<LiveRegion>, Range<u64>, usize)>,
522 opts: StreamOptions,
523 pace: Option<Pace>,
524 open: bool,
525 ) -> Result<ReadStream<P>, Error> {
526 let none = ErrorContext::default();
527 if !matches!(opts.class, IoClass::Background | IoClass::Read) {
528 return Err(not_written(NotWrittenCause::InvalidOptions, none));
529 }
530 let explicit = match (opts.depth, opts.lend) {
531 (None, None) => None,
532 (d, l) => {
533 let depth = d.or(l).unwrap_or(1);
534 let lend = l.or(d).unwrap_or(1);
535 if depth == 0 || depth > MAX_DEPTH as usize || lend == 0 {
536 return Err(not_written(NotWrittenCause::InvalidOptions, none));
537 }
538 Some((depth, lend))
539 }
540 };
541 let home = Arc::new(Mutex::new(FreeList::new(Vec::new())));
542 let block = self.block();
543 let c = parts
545 .first()
546 .map_or(none, |(r, w, _)| ctx_range(r, w.start, w.end));
547 let parts: Vec<Part> = parts
548 .into_iter()
549 .filter(|(_, want, _)| !want.is_empty())
550 .map(|(r, want, index)| Part {
551 r,
552 want,
553 index,
554 loc: Locator::Map,
555 })
556 .collect();
557 let (next, end) = parts.first().map_or((0, 0), |p| {
558 (
559 p.want.start / block * block,
560 p.want.end.div_ceil(block) * block,
561 )
562 });
563 let mut s = Core {
564 lane: None,
565 store: self.clone(),
566 parts,
567 cur: 0,
568 mark: None,
569 ring: Vec::new(),
570 submitted: 0,
571 delivered: 0,
572 in_flight: 0,
573 pending: VecDeque::new(),
574 piece: None,
575 scratch: None,
576 home,
577 lend: 0,
578 next,
579 end,
580 req: 0,
581 pace: None,
582 ended: false,
583 finished: false,
584 wait_failed: None,
585 bisected: (0, 0),
586 collect_failed: None,
587 _thread: PhantomData,
588 };
589 let req = match (opts.class, &pace) {
590 (IoClass::Read, _) => self.chunk(),
591 (_, Some(p)) => p.piece().min(self.chunk()),
592 (_, None) => (self.inner.qos.background_io() as usize).min(self.chunk()),
593 };
594 let pool = &self.inner.pool;
596 let class = pool.class_len(req).unwrap_or(req);
597 let cap = pool
598 .background_holdings()
599 .iter()
600 .find(|(size, _, _)| *size == class)
601 .map_or(0, |&(_, _, cap)| cap as usize);
602 let exhausted = || not_written(NotWrittenCause::PoolExhausted, c);
603 let explicit = match explicit {
605 Some((depth, lend)) => match depth.checked_add(lend).and_then(|n| n.checked_add(1)) {
606 Some(n) if n <= cap => Some((depth, lend, n)),
607 _ => return Err(not_written(NotWrittenCause::InvalidOptions, none)),
608 },
609 None => None,
610 };
611 if s.parts.is_empty() && !open {
612 s.ended = true;
614 s.finished = true;
615 return Ok(ReadStream::new(s));
616 }
617 let (depth, lend, mut bufs) = match explicit {
618 Some((depth, lend, n)) => {
619 let bufs: Vec<IoBuf> = (0..n)
620 .map_while(|_| pool.take_capped(req, class).ok())
621 .collect();
622 if bufs.len() < n {
623 return Err(exhausted());
624 }
625 (depth, lend, bufs)
626 }
627 None => {
628 let most = DEFAULT_DEPTH.min(cap.saturating_sub(1) / 2) * 2 + 1;
629 let bufs: Vec<IoBuf> = (0..most)
630 .map_while(|_| pool.take_capped(req, class).ok())
631 .collect();
632 if bufs.len() < 3 {
633 return Err(exhausted());
634 }
635 let depth = (bufs.len() - 1) / 2;
636 (depth, bufs.len() - 1 - depth, bufs)
637 }
638 };
639 let queue = self.thread_queue(depth as u32)?;
640 let mark = Arc::new(LaneMark::new());
641 s.lane = Some(owned_lane(queue, &mark, depth));
642 s.mark = Some(self.register_submitter(&mark));
643 s.scratch = bufs.pop();
644 lock(&s.home).bufs = bufs;
645 s.ring = (0..depth).map(|_| None).collect();
646 s.lend = lend;
647 s.req = req;
648 s.pace = pace;
649 Ok(ReadStream::new(s))
650 }
651}
652
653impl<P: Platform> ReadStream<P>
654where
655 P::Queue: Send,
656{
657 fn new(core: Core<P>) -> Self {
660 let pace = core.pace.clone();
661 let core = Rc::new(RefCell::new(core));
662 let _part = pace.map(|p| {
663 let weak: Weak<RefCell<Core<P>>> = Rc::downgrade(&core);
664 p.register(weak)
665 });
666 Self { core, _part }
667 }
668
669 pub fn cancel(self) -> Result<(), Error> {
678 let mut c = self.core.borrow_mut();
679 c.finish();
680 match c.wait_failed.take() {
681 Some(raw) => Err(c.read_error(raw)),
682 None => Ok(()),
683 }
684 }
685
686 pub(crate) fn add_part(
695 &self,
696 r: &Arc<LiveRegion>,
697 want: Range<u64>,
698 loc: Locator,
699 ) -> Result<(), Error> {
700 let mut c = self.core.borrow_mut();
701 if c.ended {
702 return Err(not_written(
703 NotWrittenCause::Interrupted,
704 ctx_range(r, want.start, want.end),
705 ));
706 }
707 if want.is_empty() {
708 return Ok(());
709 }
710 let block = c.store.block();
711 let first = c.parts.is_empty();
712 c.parts.push(Part {
713 r: Arc::clone(r),
714 want: want.clone(),
715 index: 0,
716 loc,
717 });
718 if first {
721 c.cur = 0;
722 c.next = want.start / block * block;
723 c.end = want.end.div_ceil(block) * block;
724 }
725 Ok(())
726 }
727
728 pub(crate) fn lend_bytes(
737 &self,
738 r: &LiveRegion,
739 offset: u64,
740 bytes: &[u8],
741 ) -> Result<Chunk, Error> {
742 let c = self.core.borrow_mut();
743 let ctx = || ctx_range(r, offset, offset + bytes.len() as u64);
744 if c.ended {
745 return Err(not_written(NotWrittenCause::Interrupted, ctx()));
746 }
747 let mut home = lock(&c.home);
748 if home.lent >= c.lend {
749 return Err(not_written(NotWrittenCause::TooManyInFlight, ctx()));
750 }
751 let Some(mut buf) = home.bufs.pop() else {
752 return Err(not_written(NotWrittenCause::TooManyInFlight, ctx()));
753 };
754 if !buf.set_len(bytes.len()) {
755 home.bufs.push(buf);
756 return Err(not_written(NotWrittenCause::TooLarge, ctx()));
757 }
758 buf.as_mut_capacity()[..bytes.len()].copy_from_slice(bytes);
759 home.lent += 1;
760 drop(home);
761 Ok(Chunk {
762 buf: Some(buf),
763 at: 0,
764 len: bytes.len(),
765 offset,
766 range: 0,
767 home: Arc::clone(&c.home),
768 })
769 }
770
771 pub(crate) fn pull(&self) -> Option<Result<Chunk, Error>> {
774 self.core.borrow_mut().next_item()
775 }
776
777 pub(crate) fn make_second(&self) {
781 let mut c = self.core.borrow_mut();
782 c.pace = c.pace.as_ref().map(Pace::second);
783 }
784
785 pub(crate) fn failure(&self) -> Option<Error> {
788 let c = self.core.try_borrow().ok()?;
789 c.collect_failed.map(|(raw, _)| c.read_error(raw))
790 }
791
792 pub(crate) fn lend(&self) -> usize {
794 self.core.borrow().lend
795 }
796
797 #[cfg(all(test, not(loom)))]
799 pub(crate) fn pace(&self) -> Option<Pace> {
800 self.core.borrow().pace.clone()
801 }
802}
803
804impl<P: Platform> Collect for RefCell<Core<P>>
805where
806 P::Queue: Send,
807{
808 fn collect_one(&self) -> Result<bool, OsError> {
809 match self.try_borrow_mut() {
812 Ok(mut c) => c.collect_for_another(false),
813 Err(_) => Ok(false),
814 }
815 }
816
817 fn collect_all(&self) -> Result<(), OsError> {
818 match self.try_borrow_mut() {
819 Ok(mut c) => c.collect_for_another(true).map(|_| ()),
820 Err(_) => Ok(()),
821 }
822 }
823
824 fn failed(&self) -> Option<(OsError, u64)> {
825 self.try_borrow().ok()?.collect_failed
826 }
827}
828
829impl<P: Platform> Core<P>
830where
831 P::Queue: Send,
832{
833 fn collect_for_another(&mut self, all: bool) -> Result<bool, OsError> {
839 if self.finished {
841 return Ok(false);
842 }
843 if let Some((raw, _)) = self.collect_failed {
844 return Err(raw);
845 }
846 if self.in_flight == 0 {
847 return Ok(false);
848 }
849 let waited = if all {
850 self.collect_all()
851 } else {
852 self.wait_one()
853 };
854 match waited {
855 Ok(()) => Ok(true),
856 Err(raw) => {
857 let lost = self
858 .ring
859 .iter()
860 .flatten()
861 .filter(|r| r.done.is_none())
862 .map(|r| r.start)
863 .min()
864 .unwrap_or(self.next);
865 self.collect_failed = Some((raw, lost));
866 self.wait_failed = Some(raw);
867 for r in self.ring.iter_mut().flatten() {
868 r.charge = None;
869 }
870 Err(raw)
871 }
872 }
873 }
874
875 fn ctx_of(&self, i: usize) -> ErrorContext {
877 match self.parts.get(i) {
878 Some(p) => ctx_range(&p.r, p.want.start, p.want.end),
879 None => ErrorContext::default(),
880 }
881 }
882
883 fn ctx(&self) -> ErrorContext {
885 self.ctx_of(self.cur)
886 }
887
888 fn read_error_of(&self, i: usize, raw: OsError) -> Error {
889 Error::Io {
890 op: Op::Read,
891 raw,
892 ctx: self.ctx_of(i),
893 }
894 }
895
896 fn read_error(&self, raw: OsError) -> Error {
897 self.read_error_of(self.cur, raw)
898 }
899
900 fn live_of(&self, i: usize) -> Result<(), Error> {
902 match self.parts.get(i) {
903 Some(p) => p.r.gate.live().map_err(|why| retired_error(why, &p.r)),
904 None => Err(not_written(NotWrittenCause::Interrupted, self.ctx())),
905 }
906 }
907
908 fn more(&mut self) -> bool {
911 while self.next >= self.end {
912 if self.cur + 1 >= self.parts.len() {
913 return false;
914 }
915 self.cur += 1;
916 let block = self.store.block();
917 let p = &self.parts[self.cur];
918 self.next = p.want.start / block * block;
919 self.end = p.want.end.div_ceil(block) * block;
920 }
921 true
922 }
923
924 fn copying(&self) -> bool {
927 self.pending
928 .iter()
929 .any(|i| matches!(i, Item::Run { last: false, .. }))
930 }
931
932 fn depth(&self) -> usize {
933 self.ring.len()
934 }
935
936 fn refill(&mut self) -> Result<(), Error> {
939 let depth = self.depth() as u64;
940 while !self.copying() && self.submitted - self.delivered < depth && self.more() {
941 let part = self.cur;
942 self.live_of(part)?;
943 test_hook::step(Step::StreamSubmit);
944 let Some(mut buf) = lock(&self.home).bufs.pop() else {
945 break;
946 };
947 let next = self.next;
948 let len = (self.end - next)
950 .min(self.req as u64)
951 .min(self.parts[part].run_left(next)) as usize;
952 if len == 0 {
955 lock(&self.home).bufs.push(buf);
956 return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
957 }
958 let charge = match &self.pace {
961 None => None,
962 Some(p) => match p.try_charge() {
963 Some(ch) => Some(ch),
964 None if self.in_flight > 0 => {
965 lock(&self.home).bufs.push(buf);
966 break;
967 }
968 None => match p.wait_charge() {
969 Some(ch) => Some(ch),
970 None => {
971 lock(&self.home).bufs.push(buf);
972 return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
973 }
974 },
975 },
976 };
977 let _fits = buf.set_len(len);
979 let op = IoOp::Read {
980 file: &self.store.inner.file,
981 offset: self.parts[part].locate(next, len as u64),
982 buf,
983 };
984 let Some(lane) = self.lane.as_mut() else {
985 return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
986 };
987 match lane.submit(op, self.submitted) {
988 Ok(()) => {
989 let slot = (self.submitted % depth) as usize;
990 self.ring[slot] = Some(Read {
991 part,
992 start: next,
993 len,
994 done: None,
995 charge,
996 });
997 self.submitted += 1;
998 self.next += len as u64;
999 self.in_flight += 1;
1000 }
1001 Err(rejected) => {
1002 if let Some(b) = rejected.buf {
1003 lock(&self.home).bufs.push(b);
1004 }
1005 match rejected.raw {
1006 None if self.in_flight > 0 => break,
1007 None => {
1008 return Err(not_written(NotWrittenCause::QueueFull, self.ctx()));
1009 }
1010 Some(raw) => return Err(self.read_error(raw)),
1011 }
1012 }
1013 }
1014 }
1015 Ok(())
1016 }
1017
1018 fn file(&mut self) {
1021 let depth = self.depth() as u64;
1022 let Some(lane) = self.lane.as_mut() else {
1023 return;
1024 };
1025 for comp in lane.done.drain() {
1026 self.in_flight = self.in_flight.saturating_sub(1);
1027 if let Some(p) = self.ring[(comp.tag % depth) as usize].as_mut() {
1028 p.done = Some((comp.result, comp.buf));
1029 p.charge = None;
1030 }
1031 }
1032 }
1033
1034 fn reap(&mut self) {
1036 if let Some(lane) = self.lane.as_mut() {
1037 let _arrived = lane.reap();
1038 }
1039 self.file();
1040 }
1041
1042 fn wait_one(&mut self) -> Result<(), OsError> {
1044 let waited = match self.lane.as_mut() {
1045 Some(lane) => lane.wait(1).map(|_| ()),
1046 None => Ok(()),
1047 };
1048 self.file();
1049 waited
1050 }
1051
1052 fn collect_all(&mut self) -> Result<(), OsError> {
1054 while self.in_flight > 0 {
1055 self.wait_one()?;
1056 }
1057 Ok(())
1058 }
1059
1060 fn lend_chunk(&mut self, part: usize, buf: IoBuf, at: usize, len: usize, offset: u64) -> Chunk {
1063 lock(&self.home).lent += 1;
1064 Chunk {
1065 buf: Some(buf),
1066 at,
1067 len,
1068 offset,
1069 range: self.parts.get(part).map_or(0, |p| p.index),
1070 home: Arc::clone(&self.home),
1071 }
1072 }
1073
1074 fn bad(&self, part: usize, start: u64, end: u64) -> Error {
1076 Error::Corruption {
1077 kind: CorruptionKind::MediaError,
1078 ctx: match self.parts.get(part) {
1079 Some(p) => ctx_range(&p.r, start, end),
1080 None => ErrorContext::default(),
1081 },
1082 }
1083 }
1084
1085 fn queue_bad(&mut self, part: usize, start: u64, end: u64) {
1088 self.store.inner.observer.add(Counter::MediaErrors, 1);
1089 self.pending.push_back(Item::Bad { part, start, end });
1090 }
1091
1092 fn clamp(&self, part: usize, start: u64, end: u64) -> Option<(u64, u64)> {
1095 let want = &self.parts.get(part)?.want;
1096 let (s, e) = (start.max(want.start), end.min(want.end));
1097 (s < e).then_some((s, e))
1098 }
1099
1100 fn advance(&mut self) -> Result<Next, Error> {
1103 loop {
1104 self.refill()?;
1105 self.reap();
1106 if lock(&self.home).lent >= self.lend {
1110 return Ok(Some(Err(not_written(
1111 NotWrittenCause::TooManyInFlight,
1112 self.ctx(),
1113 ))));
1114 }
1115 if let Some(item) = self.pending.pop_front() {
1116 return Ok(Some(self.deliver_item(item)));
1117 }
1118 if self.delivered == self.submitted {
1119 if !self.more() {
1120 return Ok(None);
1121 }
1122 return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
1125 }
1126 let depth = self.depth() as u64;
1127 let slot = (self.delivered % depth) as usize;
1128 let waiting = self.ring[slot].as_ref().map_or(self.cur, |p| p.part);
1129 while self.ring[slot].as_ref().is_some_and(|p| p.done.is_none()) {
1130 if let Err(raw) = self.wait_one() {
1131 return Err(self.read_error_of(waiting, raw));
1132 }
1133 }
1134 let Some(Read {
1135 part,
1136 start,
1137 len,
1138 done: Some((result, buf)),
1139 ..
1140 }) = self.ring[slot].take()
1141 else {
1142 return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
1143 };
1144 self.delivered += 1;
1145 test_hook::step(Step::StreamCompleted);
1146 if let Err(e) = self.live_of(part) {
1149 if let Some(b) = buf {
1150 lock(&self.home).bufs.push(b);
1151 }
1152 return Err(e);
1153 }
1154 match (result, buf) {
1155 (Ok(n), Some(b)) => {
1156 let n = n.min(len);
1157 let obs = &self.store.inner.observer;
1158 obs.add(Counter::Reads, 1);
1159 obs.add(Counter::ReadBytes, n as u64);
1160 if let Some((s, e)) = self.clamp(part, start + n as u64, start + len as u64) {
1162 self.queue_bad(part, s, e);
1163 }
1164 match self.clamp(part, start, start + n as u64) {
1165 Some((s, e)) => {
1166 let at = (s - start) as usize;
1167 let chunk = self.lend_chunk(part, b, at, (e - s) as usize, s);
1168 return Ok(Some(Ok(chunk)));
1169 }
1170 None => lock(&self.home).bufs.push(b),
1171 }
1172 }
1173 (Err(raw), Some(b)) if is_media_error(raw) => {
1174 self.bisect(part, start, len as u64, b)?;
1175 }
1176 (Err(raw), b) => {
1177 if let Some(b) = b {
1178 lock(&self.home).bufs.push(b);
1179 }
1180 return Err(self.read_error_of(part, raw));
1181 }
1182 (Ok(_), None) => {
1185 return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
1186 }
1187 }
1188 }
1189 }
1190
1191 fn deliver_item(&mut self, item: Item) -> Result<Chunk, Error> {
1193 match item {
1194 Item::Bad { part, start, end } => Err(self.bad(part, start, end)),
1195 Item::Run {
1196 part,
1197 start,
1198 at,
1199 len,
1200 last: true,
1201 } => match self.piece.take() {
1202 Some(b) => Ok(self.lend_chunk(part, b, at, len, start)),
1203 None => Err(not_written(NotWrittenCause::Interrupted, self.ctx())),
1204 },
1205 Item::Run {
1206 part,
1207 start,
1208 at,
1209 len,
1210 ..
1211 } => {
1212 let free = lock(&self.home).bufs.pop();
1213 let (Some(mut buf), Some(piece)) = (free, self.piece.as_ref()) else {
1214 self.pending.push_front(item);
1218 return Err(not_written(NotWrittenCause::TooManyInFlight, self.ctx()));
1219 };
1220 let _fits = buf.set_len(len);
1221 buf.as_mut_capacity()[..len].copy_from_slice(&piece.as_slice()[at..at + len]);
1222 Ok(self.lend_chunk(part, buf, 0, len, start))
1223 }
1224 }
1225 }
1226
1227 fn bisect(&mut self, part: usize, start: u64, len: u64, mut piece: IoBuf) -> Result<(), Error> {
1231 if let Err(raw) = self.collect_all() {
1234 lock(&self.home).bufs.push(piece);
1235 return Err(self.read_error_of(part, raw));
1236 }
1237 let mut scratch = self.scratch.take();
1238 let mut found: Vec<(u64, u64, bool)> = Vec::new();
1239 let halves = self.halves(
1240 part,
1241 start,
1242 start,
1243 len,
1244 &mut piece,
1245 &mut scratch,
1246 &mut found,
1247 );
1248 self.scratch = scratch;
1249 if let Err(e) = halves {
1250 self.bisected = (0, 0);
1251 lock(&self.home).bufs.push(piece);
1252 return Err(e);
1253 }
1254 test_hook::step(Step::StreamBisected);
1255 let (reads, bytes) = std::mem::take(&mut self.bisected);
1256 if let Err(e) = self.live_of(part) {
1257 lock(&self.home).bufs.push(piece);
1258 return Err(e);
1259 }
1260 let obs = &self.store.inner.observer;
1261 obs.add(Counter::Reads, reads);
1262 obs.add(Counter::ReadBytes, bytes);
1263 let mut merged: Vec<(u64, u64, bool)> = Vec::new();
1266 for (s, e, good) in found {
1267 match merged.last_mut() {
1268 Some((_, le, lg)) if *lg == good && *le == s => *le = e,
1269 _ => merged.push((s, e, good)),
1270 }
1271 }
1272 let items: Vec<Item> = merged
1273 .into_iter()
1274 .filter_map(|(s, e, good)| {
1275 let (s, e) = self.clamp(part, s, e)?;
1276 Some(if good {
1277 Item::Run {
1278 part,
1279 start: s,
1280 at: (s - start) as usize,
1281 len: (e - s) as usize,
1282 last: false,
1283 }
1284 } else {
1285 Item::Bad {
1286 part,
1287 start: s,
1288 end: e,
1289 }
1290 })
1291 })
1292 .collect();
1293 let last_run = items.iter().rposition(|i| matches!(i, Item::Run { .. }));
1294 for (i, item) in items.into_iter().enumerate() {
1295 match item {
1296 Item::Run {
1297 part,
1298 start,
1299 at,
1300 len,
1301 ..
1302 } => self.pending.push_back(Item::Run {
1303 part,
1304 start,
1305 at,
1306 len,
1307 last: Some(i) == last_run,
1308 }),
1309 Item::Bad { part, start, end } => self.queue_bad(part, start, end),
1310 }
1311 }
1312 match last_run {
1313 Some(_) => self.piece = Some(piece),
1315 None => lock(&self.home).bufs.push(piece),
1317 }
1318 Ok(())
1319 }
1320
1321 #[allow(clippy::too_many_arguments)]
1326 fn halves(
1327 &mut self,
1328 part: usize,
1329 base: u64,
1330 at: u64,
1331 len: u64,
1332 piece: &mut IoBuf,
1333 scratch: &mut Option<IoBuf>,
1334 found: &mut Vec<(u64, u64, bool)>,
1335 ) -> Result<(), Error> {
1336 let sector = self.store.sector();
1337 let block = self.store.block();
1338 let Some(mut buf) = scratch.take() else {
1339 return Err(not_written(NotWrittenCause::Interrupted, self.ctx_of(part)));
1340 };
1341 let _fits = buf.set_len(len as usize);
1343 let admitted = match &self.pace {
1344 Some(p) => Some(p.admit()?),
1345 None => None,
1346 };
1347 let op = IoOp::Read {
1348 file: &self.store.inner.file,
1349 offset: self.parts[part].locate(at, len),
1350 buf,
1351 };
1352 let Some(lane) = self.lane.as_mut() else {
1353 return Err(not_written(NotWrittenCause::Interrupted, self.ctx_of(part)));
1354 };
1355 let done = run(lane, op);
1356 drop(admitted);
1358 let (result, back) = match done {
1359 Ok(Ran::Done { buf, result }) => (result, buf),
1360 Ok(Ran::Abandoned(raw)) => return Err(self.read_error_of(part, raw)),
1361 Err((buf, Some(raw))) => {
1362 *scratch = buf;
1363 return Err(self.read_error_of(part, raw));
1364 }
1365 Err((buf, None)) => {
1366 *scratch = buf;
1367 return Err(not_written(NotWrittenCause::QueueFull, self.ctx_of(part)));
1368 }
1369 };
1370 let Some(b) = back else {
1371 return Err(match result {
1372 Err(raw) => self.read_error_of(part, raw),
1373 Ok(_) => not_written(NotWrittenCause::Interrupted, self.ctx_of(part)),
1374 });
1375 };
1376 match result {
1377 Ok(n) => {
1378 let n = (n as u64).min(len);
1379 let place = (at - base) as usize;
1380 piece.as_mut_capacity()[place..place + n as usize]
1381 .copy_from_slice(&b.as_slice()[..n as usize]);
1382 *scratch = Some(b);
1383 self.bisected.0 += 1;
1385 self.bisected.1 += n;
1386 if n > 0 {
1387 found.push((at, at + n, true));
1388 }
1389 if n < len {
1390 found.push((at + n, at + len, false));
1391 }
1392 Ok(())
1393 }
1394 Err(raw) if is_media_error(raw) => {
1395 *scratch = Some(b);
1396 if len <= sector {
1397 found.push((at, at + len, false));
1398 return Ok(());
1399 }
1400 let unit = if len > block { block } else { sector };
1403 let half = (len / unit).div_ceil(2) * unit;
1404 self.halves(part, base, at, half, piece, scratch, found)?;
1405 self.halves(part, base, at + half, len - half, piece, scratch, found)
1406 }
1407 Err(_) if len < block => {
1410 *scratch = Some(b);
1411 found.push((at, at + len, false));
1412 Ok(())
1413 }
1414 Err(raw) => {
1415 *scratch = Some(b);
1416 Err(self.read_error_of(part, raw))
1417 }
1418 }
1419 }
1420
1421 fn finish(&mut self) {
1426 if self.finished {
1427 return;
1428 }
1429 self.finished = true;
1430 self.ended = true;
1431 if self.wait_failed.is_none() {
1432 if let Err(raw) = self.collect_all() {
1433 self.wait_failed = Some(raw);
1434 }
1435 }
1436 self.ring.iter_mut().for_each(|slot| *slot = None);
1438 self.pending.clear();
1439 self.piece = None;
1440 self.scratch = None;
1441 let bufs = lock(&self.home).close();
1442 drop(bufs);
1443 }
1444}
1445
1446impl<P: Platform> Iterator for ReadStream<P>
1447where
1448 P::Queue: Send,
1449{
1450 type Item = Result<Chunk, Error>;
1451
1452 fn next(&mut self) -> Option<Self::Item> {
1453 self.core.borrow_mut().next_item()
1454 }
1455}
1456
1457impl<P: Platform> Core<P>
1458where
1459 P::Queue: Send,
1460{
1461 fn next_item(&mut self) -> Option<Result<Chunk, Error>> {
1463 if self.ended {
1464 return None;
1465 }
1466 if let Some((raw, _)) = self.collect_failed {
1469 self.finish();
1470 return Some(Err(self.read_error(raw)));
1471 }
1472 match self.advance() {
1473 Ok(Some(item)) => Some(item),
1474 Ok(None) => {
1475 self.finish();
1476 None
1477 }
1478 Err(e) => {
1479 self.finish();
1480 Some(Err(e))
1481 }
1482 }
1483 }
1484}
1485
1486impl<P: Platform> Drop for Core<P>
1487where
1488 P::Queue: Send,
1489{
1490 fn drop(&mut self) {
1491 self.finish();
1492 if let Some(id) = self.mark.take() {
1493 self.store.unregister_submitter(id);
1494 }
1495 }
1496}
1497
1498#[cfg(all(test, not(loom)))]
1499mod tests {
1500 use super::*;
1501
1502 #[test]
1503 fn test_a_chunk_is_send() {
1504 fn send<T: Send>() {}
1505 send::<Chunk>();
1506 }
1507
1508 #[test]
1509 fn test_a_closed_list_hands_buffers_back() {
1510 let mut l = FreeList::new(vec![1u32, 2]);
1511 l.lent = 1;
1512 assert_eq!(l.give_back(3), None);
1513 assert_eq!(l.close(), vec![1, 2, 3]);
1514 l.lent = 1;
1515 assert_eq!(l.give_back(4), Some(4));
1516 assert_eq!(l.lent, 0);
1517 }
1518}
1519
1520#[cfg(all(test, not(loom)))]
1523mod collect_tests {
1524 #![allow(clippy::unwrap_used, clippy::expect_used, unused_results)]
1525
1526 use std::cell::RefCell;
1527 use std::path::Path;
1528 use std::rc::Rc;
1529
1530 use store_io_sim::{SimConfig, SimPlatform};
1531
1532 use super::*;
1533 use crate::{AppendRegion, StoreOptions};
1534
1535 const LEN: usize = 4 << 20;
1536
1537 fn setup(
1541 seed: u64,
1542 ) -> (
1543 SimPlatform,
1544 Store<SimPlatform>,
1545 AppendRegion<SimPlatform>,
1546 Vec<u8>,
1547 ) {
1548 let p = SimPlatform::new(SimConfig::volatile(seed));
1549 let opts = StoreOptions {
1550 background_unpaced: 4,
1551 ..StoreOptions::default()
1552 };
1553 let s = Store::create(p.clone(), Path::new("/s"), opts).unwrap();
1554 let log = s.provision_append_region("log", LEN as u64).unwrap();
1555 let data: Vec<u8> = (0..LEN).map(|i| (i % 251) as u8).collect();
1556 let t = log.append(&data).unwrap();
1557 let _receipt = log.sync_through(&t).unwrap();
1558 p.with_world(|w| w.advance_clock_ns(10_000_000_000));
1559 assert!(!s.qos().active);
1560 (p, s, log, data)
1561 }
1562
1563 fn take(stream: &mut ReadStream<SimPlatform>, out: &mut Vec<u8>) {
1565 let c = stream.next().unwrap().unwrap();
1566 assert_eq!(c.offset() as usize, out.len());
1567 out.extend_from_slice(c.bytes());
1568 }
1569
1570 fn holding_the_cap(p: &SimPlatform, stream: &mut ReadStream<SimPlatform>, out: &mut Vec<u8>) {
1573 take(stream, out);
1574 p.with_world(|w| w.hold_reaps(1));
1575 take(stream, out);
1576 assert_eq!(stream.pace().unwrap().held_here(), 4, "the call's cap");
1577 }
1578
1579 #[test]
1583 fn test_another_part_collects_a_stream_s_reads() {
1584 let (p, _s, log, data) = setup(70);
1585 let mut stream = log.read_stream(0..LEN as u64).unwrap();
1586 let mut out = Vec::new();
1587 holding_the_cap(&p, &mut stream, &mut out);
1588 let pace = stream.pace().unwrap();
1589 let piece = pace.admit().unwrap();
1590 assert_eq!(pace.held_here(), 1, "the stream's reads collected");
1591 drop(piece);
1592 while out.len() < LEN {
1593 take(&mut stream, &mut out);
1594 }
1595 assert!(stream.next().is_none());
1596 assert!(out == data);
1597 }
1598
1599 #[test]
1601 fn test_a_paced_part_collects_all_of_a_stream_s_reads() {
1602 let (p, s, log, data) = setup(73);
1603 let mut stream = log.read_stream(0..LEN as u64).unwrap();
1604 let mut out = Vec::new();
1605 holding_the_cap(&p, &mut stream, &mut out);
1606 let hold = s.commit_hold();
1607 let pace = stream.pace().unwrap();
1608 let piece = pace.admit().unwrap();
1609 assert_eq!(pace.held_here(), 1);
1610 drop(piece);
1611 drop(hold);
1612 p.with_world(|w| w.advance_clock_ns(10_000_000_000));
1613 while out.len() < LEN {
1614 take(&mut stream, &mut out);
1615 }
1616 assert!(out == data);
1617 }
1618
1619 #[test]
1621 fn test_a_stream_with_nothing_in_flight_is_not_waited_on() {
1622 let (p, _s, log, _data) = setup(74);
1623 let mut stream = log.read_stream(0..LEN as u64).unwrap();
1624 let mut out = Vec::new();
1625 take(&mut stream, &mut out);
1626 let pace = stream.pace().unwrap();
1627 assert_eq!(pace.held_here(), 0, "the sim's wait completed every read");
1628 let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
1629 let other = Rc::new(Other(RefCell::new(held)));
1630 let weak: Weak<Other> = Rc::downgrade(&other);
1631 let _reg = pace.register(weak);
1632 let waits = p.with_world(|w| w.waits());
1633 let _piece = pace.admit().unwrap();
1634 assert_eq!(p.with_world(|w| w.waits()), waits);
1635 assert_eq!(other.0.borrow().len(), 3);
1636 }
1637
1638 struct Other(RefCell<Vec<Held>>);
1640
1641 impl Collect for Other {
1642 fn collect_one(&self) -> Result<bool, OsError> {
1643 Ok(self.0.borrow_mut().pop().is_some())
1644 }
1645
1646 fn collect_all(&self) -> Result<(), OsError> {
1647 self.0.borrow_mut().clear();
1648 Ok(())
1649 }
1650
1651 fn failed(&self) -> Option<(OsError, u64)> {
1652 None
1653 }
1654 }
1655
1656 #[test]
1660 fn test_a_stream_asking_skips_itself_and_collects_the_other_part() {
1661 let (_p, _s, log, data) = setup(71);
1662 let mut stream = log.read_stream(0..LEN as u64).unwrap();
1663 let pace = stream.pace().unwrap();
1664 let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
1665 let other = Rc::new(Other(RefCell::new(held)));
1666 let weak: Weak<Other> = Rc::downgrade(&other);
1667 let _reg = pace.register(weak);
1668 let mut out = Vec::new();
1669 while out.len() < LEN {
1670 take(&mut stream, &mut out);
1671 }
1672 assert!(stream.next().is_none());
1673 assert!(out == data);
1674 assert!(
1675 other.0.borrow().len() < 4,
1676 "the other part's pieces collected"
1677 );
1678 }
1679
1680 #[test]
1683 fn test_a_paced_stream_asking_skips_itself_and_collects_the_other_part() {
1684 let (_p, s, log, data) = setup(75);
1685 let mut stream = log.read_stream(0..LEN as u64).unwrap();
1686 let pace = stream.pace().unwrap();
1687 let held = (0..2).map(|_| pace.try_charge().unwrap()).collect();
1688 let other = Rc::new(Other(RefCell::new(held)));
1689 let weak: Weak<Other> = Rc::downgrade(&other);
1690 let _reg = pace.register(weak);
1691 let _hold = s.commit_hold();
1692 let mut out = Vec::new();
1693 while out.len() < LEN {
1694 take(&mut stream, &mut out);
1695 }
1696 assert!(out == data);
1697 assert!(other.0.borrow().is_empty());
1698 }
1699
1700 #[test]
1705 fn test_a_failed_collection_ends_the_stream_with_its_error() {
1706 let (p, _s, log, _data) = setup(72);
1707 let mut stream = log.read_stream(0..LEN as u64).unwrap();
1708 let mut out = Vec::new();
1709 holding_the_cap(&p, &mut stream, &mut out);
1710 let pace = stream.pace().unwrap();
1711 p.with_world(|w| {
1712 let next = w.waits() + 1;
1713 w.faults_mut().fail_wait = Some((next, 5));
1714 });
1715 assert!(pace.admit().is_err());
1716 assert_eq!(pace.held_here(), 0, "its charges dropped");
1717 let lost = stream.core.failed().map(|(_, at)| at);
1719 assert_eq!(lost, Some(4 * (out.len() as u64 / 2)));
1720 let waits = p.with_world(|w| w.waits());
1721 let again = pace.try_charge().unwrap();
1723 let more: Vec<_> = (0..3).map(|_| pace.try_charge().unwrap()).collect();
1724 assert!(pace.admit().is_err());
1725 assert_eq!(p.with_world(|w| w.waits()), waits);
1726 drop((again, more));
1727 assert!(matches!(stream.next(), Some(Err(Error::Io { .. }))));
1728 assert!(stream.next().is_none());
1729 let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
1731 let other = Rc::new(Other(RefCell::new(held)));
1732 let weak: Weak<Other> = Rc::downgrade(&other);
1733 let _reg = pace.register(weak);
1734 let _piece = pace.admit().unwrap();
1735 drop(stream);
1736 assert_eq!(p.with_world(|w| w.waits()), waits, "never waited on again");
1737 }
1738}
1739
1740#[cfg(all(test, loom))]
1741mod loom_tests {
1742 use super::*;
1743 use loom::thread;
1744
1745 #[test]
1749 fn loom_a_give_back_against_a_close_strands_nothing() {
1750 loom::model(|| {
1751 let home = Arc::new(Mutex::new(FreeList::new(vec![1u32])));
1752 lock(&home).lent = 1;
1753 let h = Arc::clone(&home);
1754 let t = thread::spawn(move || lock(&h).give_back(2));
1755 let drained = lock(&home).close();
1756 let back = t.join().unwrap_or(None);
1757 assert!(
1760 lock(&home).bufs.is_empty(),
1761 "a buffer pushed after the close"
1762 );
1763 assert_eq!(drained.len() + usize::from(back.is_some()), 2);
1764 });
1765 }
1766}