use std::cell::RefCell;
use std::collections::VecDeque;
use std::marker::PhantomData;
use std::ops::Range;
use std::rc::{Rc, Weak};
use std::sync::{Arc, PoisonError};
#[cfg(loom)]
use loom::sync::{Mutex, MutexGuard};
#[cfg(not(loom))]
use std::sync::{Mutex, MutexGuard};
use store_io_core::IoClass;
use store_io_core::errno::is_media_error;
use store_io_core::error::{CorruptionKind, Error, ErrorContext, NotWrittenCause, Op, OsError};
use store_io_core::observe::Counter;
use store_io_platform::{IoBuf, IoOp, Platform, RawResult};
use crate::exec::{Lane, LaneMark, Ran, owned_lane, run};
use crate::io::{ctx_range, not_written};
use crate::lifecycle::retired_error;
use crate::pace::{Held, Pace, Registered};
use crate::qos::Collect;
use crate::store::{LiveRegion, Store};
use crate::submitter::MAX_DEPTH;
use crate::test_hook::{self, Step};
const DEFAULT_DEPTH: usize = 8;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub struct StreamOptions {
depth: Option<usize>,
lend: Option<usize>,
class: IoClass,
}
impl Default for StreamOptions {
fn default() -> Self {
Self {
depth: None,
lend: None,
class: IoClass::Background,
}
}
}
impl StreamOptions {
#[must_use]
pub fn depth(mut self, depth: usize) -> Self {
self.depth = Some(depth);
self
}
#[must_use]
pub fn lend(mut self, lend: usize) -> Self {
self.lend = Some(lend);
self
}
#[must_use]
pub fn class(mut self, class: IoClass) -> Self {
self.class = class;
self
}
}
pub(crate) struct FreeList<B> {
closed: bool,
bufs: Vec<B>,
lent: usize,
}
impl<B> FreeList<B> {
pub(crate) fn new(bufs: Vec<B>) -> Self {
Self {
closed: false,
bufs,
lent: 0,
}
}
pub(crate) fn give_back(&mut self, b: B) -> Option<B> {
self.lent = self.lent.saturating_sub(1);
if self.closed {
Some(b)
} else {
self.bufs.push(b);
None
}
}
pub(crate) fn close(&mut self) -> Vec<B> {
self.closed = true;
std::mem::take(&mut self.bufs)
}
}
fn lock<B>(m: &Mutex<FreeList<B>>) -> MutexGuard<'_, FreeList<B>> {
m.lock().unwrap_or_else(PoisonError::into_inner)
}
pub struct Chunk {
buf: Option<IoBuf>,
at: usize,
len: usize,
offset: u64,
range: usize,
home: Arc<Mutex<FreeList<IoBuf>>>,
}
impl Chunk {
#[must_use]
pub fn offset(&self) -> u64 {
self.offset
}
#[must_use]
pub fn range(&self) -> usize {
self.range
}
#[must_use]
pub fn bytes(&self) -> &[u8] {
match &self.buf {
Some(b) => &b.as_slice()[self.at..self.at + self.len],
None => &[],
}
}
}
impl core::fmt::Debug for Chunk {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Chunk")
.field("offset", &self.offset)
.field("len", &self.len)
.field("range", &self.range)
.finish_non_exhaustive()
}
}
impl Drop for Chunk {
fn drop(&mut self) {
if let Some(b) = self.buf.take() {
let back = lock(&self.home).give_back(b);
drop(back);
}
}
}
struct Read {
part: usize,
start: u64,
len: usize,
done: Option<(RawResult<usize>, Option<IoBuf>)>,
charge: Option<Held>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Item {
Run {
part: usize,
start: u64,
at: usize,
len: usize,
last: bool,
},
Bad { part: usize, start: u64, end: u64 },
}
struct Part {
r: Arc<LiveRegion>,
want: Range<u64>,
index: usize,
loc: Locator,
}
pub(crate) enum Locator {
Map,
Ranges(Vec<(u64, u64)>),
}
impl Part {
fn run_left(&self, at: u64) -> u64 {
match &self.loc {
Locator::Map => self.r.map.run_left(at),
Locator::Ranges(ranges) => {
let mut base = 0u64;
for &(_, len) in ranges {
if at < base + len {
return base + len - at;
}
base += len;
}
0
}
}
}
fn locate(&self, at: u64, len: u64) -> u64 {
match &self.loc {
Locator::Map => self.r.map.locate(at, len),
Locator::Ranges(ranges) => {
let mut base = 0u64;
for &(file, n) in ranges {
if at < base + n {
return file + (at - base);
}
base += n;
}
u64::MAX
}
}
}
}
pub struct ReadStream<P: Platform>
where
P::Queue: Send,
{
core: Rc<RefCell<Core<P>>>,
_part: Option<Registered>,
}
struct Core<P: Platform>
where
P::Queue: Send,
{
lane: Option<Lane<P::Queue>>,
store: Store<P>,
parts: Vec<Part>,
cur: usize,
mark: Option<u64>,
ring: Vec<Option<Read>>,
submitted: u64,
delivered: u64,
in_flight: usize,
pending: VecDeque<Item>,
piece: Option<IoBuf>,
scratch: Option<IoBuf>,
home: Arc<Mutex<FreeList<IoBuf>>>,
lend: usize,
next: u64,
end: u64,
req: usize,
pace: Option<Pace>,
ended: bool,
finished: bool,
wait_failed: Option<OsError>,
bisected: (u64, u64),
collect_failed: Option<(OsError, u64)>,
_thread: PhantomData<*const ()>,
}
impl<P: Platform> core::fmt::Debug for ReadStream<P>
where
P::Queue: Send,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
let mut d = f.debug_struct("ReadStream");
if let Ok(c) = self.core.try_borrow() {
let _d = d
.field("parts", &c.parts.len())
.field("cur", &c.cur)
.field("next", &c.next)
.field("ended", &c.ended);
}
d.finish_non_exhaustive()
}
}
type Next = Option<Result<Chunk, Error>>;
pub struct StreamRange<'a, P: Platform> {
store: &'a Store<P>,
r: Arc<LiveRegion>,
want: Range<u64>,
}
impl<'a, P: Platform> StreamRange<'a, P> {
pub(crate) fn new(store: &'a Store<P>, r: &Arc<LiveRegion>, want: Range<u64>) -> Self {
Self {
store,
r: Arc::clone(r),
want,
}
}
}
impl<P: Platform> core::fmt::Debug for StreamRange<'_, P> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("StreamRange")
.field("region", &self.r.id)
.field("want", &self.want)
.finish_non_exhaustive()
}
}
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub fn prefetch(
&self,
ranges: &[StreamRange<'_, P>],
opts: StreamOptions,
) -> Result<ReadStream<P>, Error> {
for sr in ranges {
if !Arc::ptr_eq(&sr.store.inner, &self.inner) {
return Err(not_written(
NotWrittenCause::ForeignPosition,
ctx_range(&sr.r, sr.want.start, sr.want.end),
));
}
sr.r.gate.live().map_err(|why| retired_error(why, &sr.r))?;
}
let parts = ranges
.iter()
.enumerate()
.map(|(i, sr)| (Arc::clone(&sr.r), sr.want.clone(), i))
.collect();
self.read_stream_of(parts, opts)
}
}
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub(crate) fn read_stream_of(
&self,
parts: Vec<(Arc<LiveRegion>, Range<u64>, usize)>,
opts: StreamOptions,
) -> Result<ReadStream<P>, Error> {
let pace = match opts.class {
IoClass::Read => None,
_ => self.inner.domain.new_pace(),
};
self.stream_of(parts, opts, pace, false)
}
pub(crate) fn export_stream(
&self,
depth: Option<usize>,
pace: Option<Pace>,
) -> Result<ReadStream<P>, Error> {
let mut opts = StreamOptions::default();
if let Some(d) = depth {
opts = opts.depth(d);
}
self.stream_of(Vec::new(), opts, pace, true)
}
fn stream_of(
&self,
parts: Vec<(Arc<LiveRegion>, Range<u64>, usize)>,
opts: StreamOptions,
pace: Option<Pace>,
open: bool,
) -> Result<ReadStream<P>, Error> {
let none = ErrorContext::default();
if !matches!(opts.class, IoClass::Background | IoClass::Read) {
return Err(not_written(NotWrittenCause::InvalidOptions, none));
}
let explicit = match (opts.depth, opts.lend) {
(None, None) => None,
(d, l) => {
let depth = d.or(l).unwrap_or(1);
let lend = l.or(d).unwrap_or(1);
if depth == 0 || depth > MAX_DEPTH as usize || lend == 0 {
return Err(not_written(NotWrittenCause::InvalidOptions, none));
}
Some((depth, lend))
}
};
let home = Arc::new(Mutex::new(FreeList::new(Vec::new())));
let block = self.block();
let c = parts
.first()
.map_or(none, |(r, w, _)| ctx_range(r, w.start, w.end));
let parts: Vec<Part> = parts
.into_iter()
.filter(|(_, want, _)| !want.is_empty())
.map(|(r, want, index)| Part {
r,
want,
index,
loc: Locator::Map,
})
.collect();
let (next, end) = parts.first().map_or((0, 0), |p| {
(
p.want.start / block * block,
p.want.end.div_ceil(block) * block,
)
});
let mut s = Core {
lane: None,
store: self.clone(),
parts,
cur: 0,
mark: None,
ring: Vec::new(),
submitted: 0,
delivered: 0,
in_flight: 0,
pending: VecDeque::new(),
piece: None,
scratch: None,
home,
lend: 0,
next,
end,
req: 0,
pace: None,
ended: false,
finished: false,
wait_failed: None,
bisected: (0, 0),
collect_failed: None,
_thread: PhantomData,
};
let req = match (opts.class, &pace) {
(IoClass::Read, _) => self.chunk(),
(_, Some(p)) => p.piece().min(self.chunk()),
(_, None) => (self.inner.qos.background_io() as usize).min(self.chunk()),
};
let pool = &self.inner.pool;
let class = pool.class_len(req).unwrap_or(req);
let cap = pool
.background_holdings()
.iter()
.find(|(size, _, _)| *size == class)
.map_or(0, |&(_, _, cap)| cap as usize);
let exhausted = || not_written(NotWrittenCause::PoolExhausted, c);
let explicit = match explicit {
Some((depth, lend)) => match depth.checked_add(lend).and_then(|n| n.checked_add(1)) {
Some(n) if n <= cap => Some((depth, lend, n)),
_ => return Err(not_written(NotWrittenCause::InvalidOptions, none)),
},
None => None,
};
if s.parts.is_empty() && !open {
s.ended = true;
s.finished = true;
return Ok(ReadStream::new(s));
}
let (depth, lend, mut bufs) = match explicit {
Some((depth, lend, n)) => {
let bufs: Vec<IoBuf> = (0..n)
.map_while(|_| pool.take_capped(req, class).ok())
.collect();
if bufs.len() < n {
return Err(exhausted());
}
(depth, lend, bufs)
}
None => {
let most = DEFAULT_DEPTH.min(cap.saturating_sub(1) / 2) * 2 + 1;
let bufs: Vec<IoBuf> = (0..most)
.map_while(|_| pool.take_capped(req, class).ok())
.collect();
if bufs.len() < 3 {
return Err(exhausted());
}
let depth = (bufs.len() - 1) / 2;
(depth, bufs.len() - 1 - depth, bufs)
}
};
let queue = self.thread_queue(depth as u32)?;
let mark = Arc::new(LaneMark::new());
s.lane = Some(owned_lane(queue, &mark, depth));
s.mark = Some(self.register_submitter(&mark));
s.scratch = bufs.pop();
lock(&s.home).bufs = bufs;
s.ring = (0..depth).map(|_| None).collect();
s.lend = lend;
s.req = req;
s.pace = pace;
Ok(ReadStream::new(s))
}
}
impl<P: Platform> ReadStream<P>
where
P::Queue: Send,
{
fn new(core: Core<P>) -> Self {
let pace = core.pace.clone();
let core = Rc::new(RefCell::new(core));
let _part = pace.map(|p| {
let weak: Weak<RefCell<Core<P>>> = Rc::downgrade(&core);
p.register(weak)
});
Self { core, _part }
}
pub fn cancel(self) -> Result<(), Error> {
let mut c = self.core.borrow_mut();
c.finish();
match c.wait_failed.take() {
Some(raw) => Err(c.read_error(raw)),
None => Ok(()),
}
}
pub(crate) fn add_part(
&self,
r: &Arc<LiveRegion>,
want: Range<u64>,
loc: Locator,
) -> Result<(), Error> {
let mut c = self.core.borrow_mut();
if c.ended {
return Err(not_written(
NotWrittenCause::Interrupted,
ctx_range(r, want.start, want.end),
));
}
if want.is_empty() {
return Ok(());
}
let block = c.store.block();
let first = c.parts.is_empty();
c.parts.push(Part {
r: Arc::clone(r),
want: want.clone(),
index: 0,
loc,
});
if first {
c.cur = 0;
c.next = want.start / block * block;
c.end = want.end.div_ceil(block) * block;
}
Ok(())
}
pub(crate) fn lend_bytes(
&self,
r: &LiveRegion,
offset: u64,
bytes: &[u8],
) -> Result<Chunk, Error> {
let c = self.core.borrow_mut();
let ctx = || ctx_range(r, offset, offset + bytes.len() as u64);
if c.ended {
return Err(not_written(NotWrittenCause::Interrupted, ctx()));
}
let mut home = lock(&c.home);
if home.lent >= c.lend {
return Err(not_written(NotWrittenCause::TooManyInFlight, ctx()));
}
let Some(mut buf) = home.bufs.pop() else {
return Err(not_written(NotWrittenCause::TooManyInFlight, ctx()));
};
if !buf.set_len(bytes.len()) {
home.bufs.push(buf);
return Err(not_written(NotWrittenCause::TooLarge, ctx()));
}
buf.as_mut_capacity()[..bytes.len()].copy_from_slice(bytes);
home.lent += 1;
drop(home);
Ok(Chunk {
buf: Some(buf),
at: 0,
len: bytes.len(),
offset,
range: 0,
home: Arc::clone(&c.home),
})
}
pub(crate) fn pull(&self) -> Option<Result<Chunk, Error>> {
self.core.borrow_mut().next_item()
}
pub(crate) fn make_second(&self) {
let mut c = self.core.borrow_mut();
c.pace = c.pace.as_ref().map(Pace::second);
}
pub(crate) fn failure(&self) -> Option<Error> {
let c = self.core.try_borrow().ok()?;
c.collect_failed.map(|(raw, _)| c.read_error(raw))
}
pub(crate) fn lend(&self) -> usize {
self.core.borrow().lend
}
#[cfg(all(test, not(loom)))]
pub(crate) fn pace(&self) -> Option<Pace> {
self.core.borrow().pace.clone()
}
}
impl<P: Platform> Collect for RefCell<Core<P>>
where
P::Queue: Send,
{
fn collect_one(&self) -> Result<bool, OsError> {
match self.try_borrow_mut() {
Ok(mut c) => c.collect_for_another(false),
Err(_) => Ok(false),
}
}
fn collect_all(&self) -> Result<(), OsError> {
match self.try_borrow_mut() {
Ok(mut c) => c.collect_for_another(true).map(|_| ()),
Err(_) => Ok(()),
}
}
fn failed(&self) -> Option<(OsError, u64)> {
self.try_borrow().ok()?.collect_failed
}
}
impl<P: Platform> Core<P>
where
P::Queue: Send,
{
fn collect_for_another(&mut self, all: bool) -> Result<bool, OsError> {
if self.finished {
return Ok(false);
}
if let Some((raw, _)) = self.collect_failed {
return Err(raw);
}
if self.in_flight == 0 {
return Ok(false);
}
let waited = if all {
self.collect_all()
} else {
self.wait_one()
};
match waited {
Ok(()) => Ok(true),
Err(raw) => {
let lost = self
.ring
.iter()
.flatten()
.filter(|r| r.done.is_none())
.map(|r| r.start)
.min()
.unwrap_or(self.next);
self.collect_failed = Some((raw, lost));
self.wait_failed = Some(raw);
for r in self.ring.iter_mut().flatten() {
r.charge = None;
}
Err(raw)
}
}
}
fn ctx_of(&self, i: usize) -> ErrorContext {
match self.parts.get(i) {
Some(p) => ctx_range(&p.r, p.want.start, p.want.end),
None => ErrorContext::default(),
}
}
fn ctx(&self) -> ErrorContext {
self.ctx_of(self.cur)
}
fn read_error_of(&self, i: usize, raw: OsError) -> Error {
Error::Io {
op: Op::Read,
raw,
ctx: self.ctx_of(i),
}
}
fn read_error(&self, raw: OsError) -> Error {
self.read_error_of(self.cur, raw)
}
fn live_of(&self, i: usize) -> Result<(), Error> {
match self.parts.get(i) {
Some(p) => p.r.gate.live().map_err(|why| retired_error(why, &p.r)),
None => Err(not_written(NotWrittenCause::Interrupted, self.ctx())),
}
}
fn more(&mut self) -> bool {
while self.next >= self.end {
if self.cur + 1 >= self.parts.len() {
return false;
}
self.cur += 1;
let block = self.store.block();
let p = &self.parts[self.cur];
self.next = p.want.start / block * block;
self.end = p.want.end.div_ceil(block) * block;
}
true
}
fn copying(&self) -> bool {
self.pending
.iter()
.any(|i| matches!(i, Item::Run { last: false, .. }))
}
fn depth(&self) -> usize {
self.ring.len()
}
fn refill(&mut self) -> Result<(), Error> {
let depth = self.depth() as u64;
while !self.copying() && self.submitted - self.delivered < depth && self.more() {
let part = self.cur;
self.live_of(part)?;
test_hook::step(Step::StreamSubmit);
let Some(mut buf) = lock(&self.home).bufs.pop() else {
break;
};
let next = self.next;
let len = (self.end - next)
.min(self.req as u64)
.min(self.parts[part].run_left(next)) as usize;
if len == 0 {
lock(&self.home).bufs.push(buf);
return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
}
let charge = match &self.pace {
None => None,
Some(p) => match p.try_charge() {
Some(ch) => Some(ch),
None if self.in_flight > 0 => {
lock(&self.home).bufs.push(buf);
break;
}
None => match p.wait_charge() {
Some(ch) => Some(ch),
None => {
lock(&self.home).bufs.push(buf);
return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
}
},
},
};
let _fits = buf.set_len(len);
let op = IoOp::Read {
file: &self.store.inner.file,
offset: self.parts[part].locate(next, len as u64),
buf,
};
let Some(lane) = self.lane.as_mut() else {
return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
};
match lane.submit(op, self.submitted) {
Ok(()) => {
let slot = (self.submitted % depth) as usize;
self.ring[slot] = Some(Read {
part,
start: next,
len,
done: None,
charge,
});
self.submitted += 1;
self.next += len as u64;
self.in_flight += 1;
}
Err(rejected) => {
if let Some(b) = rejected.buf {
lock(&self.home).bufs.push(b);
}
match rejected.raw {
None if self.in_flight > 0 => break,
None => {
return Err(not_written(NotWrittenCause::QueueFull, self.ctx()));
}
Some(raw) => return Err(self.read_error(raw)),
}
}
}
}
Ok(())
}
fn file(&mut self) {
let depth = self.depth() as u64;
let Some(lane) = self.lane.as_mut() else {
return;
};
for comp in lane.done.drain() {
self.in_flight = self.in_flight.saturating_sub(1);
if let Some(p) = self.ring[(comp.tag % depth) as usize].as_mut() {
p.done = Some((comp.result, comp.buf));
p.charge = None;
}
}
}
fn reap(&mut self) {
if let Some(lane) = self.lane.as_mut() {
let _arrived = lane.reap();
}
self.file();
}
fn wait_one(&mut self) -> Result<(), OsError> {
let waited = match self.lane.as_mut() {
Some(lane) => lane.wait(1).map(|_| ()),
None => Ok(()),
};
self.file();
waited
}
fn collect_all(&mut self) -> Result<(), OsError> {
while self.in_flight > 0 {
self.wait_one()?;
}
Ok(())
}
fn lend_chunk(&mut self, part: usize, buf: IoBuf, at: usize, len: usize, offset: u64) -> Chunk {
lock(&self.home).lent += 1;
Chunk {
buf: Some(buf),
at,
len,
offset,
range: self.parts.get(part).map_or(0, |p| p.index),
home: Arc::clone(&self.home),
}
}
fn bad(&self, part: usize, start: u64, end: u64) -> Error {
Error::Corruption {
kind: CorruptionKind::MediaError,
ctx: match self.parts.get(part) {
Some(p) => ctx_range(&p.r, start, end),
None => ErrorContext::default(),
},
}
}
fn queue_bad(&mut self, part: usize, start: u64, end: u64) {
self.store.inner.observer.add(Counter::MediaErrors, 1);
self.pending.push_back(Item::Bad { part, start, end });
}
fn clamp(&self, part: usize, start: u64, end: u64) -> Option<(u64, u64)> {
let want = &self.parts.get(part)?.want;
let (s, e) = (start.max(want.start), end.min(want.end));
(s < e).then_some((s, e))
}
fn advance(&mut self) -> Result<Next, Error> {
loop {
self.refill()?;
self.reap();
if lock(&self.home).lent >= self.lend {
return Ok(Some(Err(not_written(
NotWrittenCause::TooManyInFlight,
self.ctx(),
))));
}
if let Some(item) = self.pending.pop_front() {
return Ok(Some(self.deliver_item(item)));
}
if self.delivered == self.submitted {
if !self.more() {
return Ok(None);
}
return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
}
let depth = self.depth() as u64;
let slot = (self.delivered % depth) as usize;
let waiting = self.ring[slot].as_ref().map_or(self.cur, |p| p.part);
while self.ring[slot].as_ref().is_some_and(|p| p.done.is_none()) {
if let Err(raw) = self.wait_one() {
return Err(self.read_error_of(waiting, raw));
}
}
let Some(Read {
part,
start,
len,
done: Some((result, buf)),
..
}) = self.ring[slot].take()
else {
return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
};
self.delivered += 1;
test_hook::step(Step::StreamCompleted);
if let Err(e) = self.live_of(part) {
if let Some(b) = buf {
lock(&self.home).bufs.push(b);
}
return Err(e);
}
match (result, buf) {
(Ok(n), Some(b)) => {
let n = n.min(len);
let obs = &self.store.inner.observer;
obs.add(Counter::Reads, 1);
obs.add(Counter::ReadBytes, n as u64);
if let Some((s, e)) = self.clamp(part, start + n as u64, start + len as u64) {
self.queue_bad(part, s, e);
}
match self.clamp(part, start, start + n as u64) {
Some((s, e)) => {
let at = (s - start) as usize;
let chunk = self.lend_chunk(part, b, at, (e - s) as usize, s);
return Ok(Some(Ok(chunk)));
}
None => lock(&self.home).bufs.push(b),
}
}
(Err(raw), Some(b)) if is_media_error(raw) => {
self.bisect(part, start, len as u64, b)?;
}
(Err(raw), b) => {
if let Some(b) = b {
lock(&self.home).bufs.push(b);
}
return Err(self.read_error_of(part, raw));
}
(Ok(_), None) => {
return Err(not_written(NotWrittenCause::Interrupted, self.ctx()));
}
}
}
}
fn deliver_item(&mut self, item: Item) -> Result<Chunk, Error> {
match item {
Item::Bad { part, start, end } => Err(self.bad(part, start, end)),
Item::Run {
part,
start,
at,
len,
last: true,
} => match self.piece.take() {
Some(b) => Ok(self.lend_chunk(part, b, at, len, start)),
None => Err(not_written(NotWrittenCause::Interrupted, self.ctx())),
},
Item::Run {
part,
start,
at,
len,
..
} => {
let free = lock(&self.home).bufs.pop();
let (Some(mut buf), Some(piece)) = (free, self.piece.as_ref()) else {
self.pending.push_front(item);
return Err(not_written(NotWrittenCause::TooManyInFlight, self.ctx()));
};
let _fits = buf.set_len(len);
buf.as_mut_capacity()[..len].copy_from_slice(&piece.as_slice()[at..at + len]);
Ok(self.lend_chunk(part, buf, 0, len, start))
}
}
}
fn bisect(&mut self, part: usize, start: u64, len: u64, mut piece: IoBuf) -> Result<(), Error> {
if let Err(raw) = self.collect_all() {
lock(&self.home).bufs.push(piece);
return Err(self.read_error_of(part, raw));
}
let mut scratch = self.scratch.take();
let mut found: Vec<(u64, u64, bool)> = Vec::new();
let halves = self.halves(
part,
start,
start,
len,
&mut piece,
&mut scratch,
&mut found,
);
self.scratch = scratch;
if let Err(e) = halves {
self.bisected = (0, 0);
lock(&self.home).bufs.push(piece);
return Err(e);
}
test_hook::step(Step::StreamBisected);
let (reads, bytes) = std::mem::take(&mut self.bisected);
if let Err(e) = self.live_of(part) {
lock(&self.home).bufs.push(piece);
return Err(e);
}
let obs = &self.store.inner.observer;
obs.add(Counter::Reads, reads);
obs.add(Counter::ReadBytes, bytes);
let mut merged: Vec<(u64, u64, bool)> = Vec::new();
for (s, e, good) in found {
match merged.last_mut() {
Some((_, le, lg)) if *lg == good && *le == s => *le = e,
_ => merged.push((s, e, good)),
}
}
let items: Vec<Item> = merged
.into_iter()
.filter_map(|(s, e, good)| {
let (s, e) = self.clamp(part, s, e)?;
Some(if good {
Item::Run {
part,
start: s,
at: (s - start) as usize,
len: (e - s) as usize,
last: false,
}
} else {
Item::Bad {
part,
start: s,
end: e,
}
})
})
.collect();
let last_run = items.iter().rposition(|i| matches!(i, Item::Run { .. }));
for (i, item) in items.into_iter().enumerate() {
match item {
Item::Run {
part,
start,
at,
len,
..
} => self.pending.push_back(Item::Run {
part,
start,
at,
len,
last: Some(i) == last_run,
}),
Item::Bad { part, start, end } => self.queue_bad(part, start, end),
}
}
match last_run {
Some(_) => self.piece = Some(piece),
None => lock(&self.home).bufs.push(piece),
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn halves(
&mut self,
part: usize,
base: u64,
at: u64,
len: u64,
piece: &mut IoBuf,
scratch: &mut Option<IoBuf>,
found: &mut Vec<(u64, u64, bool)>,
) -> Result<(), Error> {
let sector = self.store.sector();
let block = self.store.block();
let Some(mut buf) = scratch.take() else {
return Err(not_written(NotWrittenCause::Interrupted, self.ctx_of(part)));
};
let _fits = buf.set_len(len as usize);
let admitted = match &self.pace {
Some(p) => Some(p.admit()?),
None => None,
};
let op = IoOp::Read {
file: &self.store.inner.file,
offset: self.parts[part].locate(at, len),
buf,
};
let Some(lane) = self.lane.as_mut() else {
return Err(not_written(NotWrittenCause::Interrupted, self.ctx_of(part)));
};
let done = run(lane, op);
drop(admitted);
let (result, back) = match done {
Ok(Ran::Done { buf, result }) => (result, buf),
Ok(Ran::Abandoned(raw)) => return Err(self.read_error_of(part, raw)),
Err((buf, Some(raw))) => {
*scratch = buf;
return Err(self.read_error_of(part, raw));
}
Err((buf, None)) => {
*scratch = buf;
return Err(not_written(NotWrittenCause::QueueFull, self.ctx_of(part)));
}
};
let Some(b) = back else {
return Err(match result {
Err(raw) => self.read_error_of(part, raw),
Ok(_) => not_written(NotWrittenCause::Interrupted, self.ctx_of(part)),
});
};
match result {
Ok(n) => {
let n = (n as u64).min(len);
let place = (at - base) as usize;
piece.as_mut_capacity()[place..place + n as usize]
.copy_from_slice(&b.as_slice()[..n as usize]);
*scratch = Some(b);
self.bisected.0 += 1;
self.bisected.1 += n;
if n > 0 {
found.push((at, at + n, true));
}
if n < len {
found.push((at + n, at + len, false));
}
Ok(())
}
Err(raw) if is_media_error(raw) => {
*scratch = Some(b);
if len <= sector {
found.push((at, at + len, false));
return Ok(());
}
let unit = if len > block { block } else { sector };
let half = (len / unit).div_ceil(2) * unit;
self.halves(part, base, at, half, piece, scratch, found)?;
self.halves(part, base, at + half, len - half, piece, scratch, found)
}
Err(_) if len < block => {
*scratch = Some(b);
found.push((at, at + len, false));
Ok(())
}
Err(raw) => {
*scratch = Some(b);
Err(self.read_error_of(part, raw))
}
}
}
fn finish(&mut self) {
if self.finished {
return;
}
self.finished = true;
self.ended = true;
if self.wait_failed.is_none() {
if let Err(raw) = self.collect_all() {
self.wait_failed = Some(raw);
}
}
self.ring.iter_mut().for_each(|slot| *slot = None);
self.pending.clear();
self.piece = None;
self.scratch = None;
let bufs = lock(&self.home).close();
drop(bufs);
}
}
impl<P: Platform> Iterator for ReadStream<P>
where
P::Queue: Send,
{
type Item = Result<Chunk, Error>;
fn next(&mut self) -> Option<Self::Item> {
self.core.borrow_mut().next_item()
}
}
impl<P: Platform> Core<P>
where
P::Queue: Send,
{
fn next_item(&mut self) -> Option<Result<Chunk, Error>> {
if self.ended {
return None;
}
if let Some((raw, _)) = self.collect_failed {
self.finish();
return Some(Err(self.read_error(raw)));
}
match self.advance() {
Ok(Some(item)) => Some(item),
Ok(None) => {
self.finish();
None
}
Err(e) => {
self.finish();
Some(Err(e))
}
}
}
}
impl<P: Platform> Drop for Core<P>
where
P::Queue: Send,
{
fn drop(&mut self) {
self.finish();
if let Some(id) = self.mark.take() {
self.store.unregister_submitter(id);
}
}
}
#[cfg(all(test, not(loom)))]
mod tests {
use super::*;
#[test]
fn test_a_chunk_is_send() {
fn send<T: Send>() {}
send::<Chunk>();
}
#[test]
fn test_a_closed_list_hands_buffers_back() {
let mut l = FreeList::new(vec![1u32, 2]);
l.lent = 1;
assert_eq!(l.give_back(3), None);
assert_eq!(l.close(), vec![1, 2, 3]);
l.lent = 1;
assert_eq!(l.give_back(4), Some(4));
assert_eq!(l.lent, 0);
}
}
#[cfg(all(test, not(loom)))]
mod collect_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, unused_results)]
use std::cell::RefCell;
use std::path::Path;
use std::rc::Rc;
use store_io_sim::{SimConfig, SimPlatform};
use super::*;
use crate::{AppendRegion, StoreOptions};
const LEN: usize = 4 << 20;
fn setup(
seed: u64,
) -> (
SimPlatform,
Store<SimPlatform>,
AppendRegion<SimPlatform>,
Vec<u8>,
) {
let p = SimPlatform::new(SimConfig::volatile(seed));
let opts = StoreOptions {
background_unpaced: 4,
..StoreOptions::default()
};
let s = Store::create(p.clone(), Path::new("/s"), opts).unwrap();
let log = s.provision_append_region("log", LEN as u64).unwrap();
let data: Vec<u8> = (0..LEN).map(|i| (i % 251) as u8).collect();
let t = log.append(&data).unwrap();
let _receipt = log.sync_through(&t).unwrap();
p.with_world(|w| w.advance_clock_ns(10_000_000_000));
assert!(!s.qos().active);
(p, s, log, data)
}
fn take(stream: &mut ReadStream<SimPlatform>, out: &mut Vec<u8>) {
let c = stream.next().unwrap().unwrap();
assert_eq!(c.offset() as usize, out.len());
out.extend_from_slice(c.bytes());
}
fn holding_the_cap(p: &SimPlatform, stream: &mut ReadStream<SimPlatform>, out: &mut Vec<u8>) {
take(stream, out);
p.with_world(|w| w.hold_reaps(1));
take(stream, out);
assert_eq!(stream.pace().unwrap().held_here(), 4, "the call's cap");
}
#[test]
fn test_another_part_collects_a_stream_s_reads() {
let (p, _s, log, data) = setup(70);
let mut stream = log.read_stream(0..LEN as u64).unwrap();
let mut out = Vec::new();
holding_the_cap(&p, &mut stream, &mut out);
let pace = stream.pace().unwrap();
let piece = pace.admit().unwrap();
assert_eq!(pace.held_here(), 1, "the stream's reads collected");
drop(piece);
while out.len() < LEN {
take(&mut stream, &mut out);
}
assert!(stream.next().is_none());
assert!(out == data);
}
#[test]
fn test_a_paced_part_collects_all_of_a_stream_s_reads() {
let (p, s, log, data) = setup(73);
let mut stream = log.read_stream(0..LEN as u64).unwrap();
let mut out = Vec::new();
holding_the_cap(&p, &mut stream, &mut out);
let hold = s.commit_hold();
let pace = stream.pace().unwrap();
let piece = pace.admit().unwrap();
assert_eq!(pace.held_here(), 1);
drop(piece);
drop(hold);
p.with_world(|w| w.advance_clock_ns(10_000_000_000));
while out.len() < LEN {
take(&mut stream, &mut out);
}
assert!(out == data);
}
#[test]
fn test_a_stream_with_nothing_in_flight_is_not_waited_on() {
let (p, _s, log, _data) = setup(74);
let mut stream = log.read_stream(0..LEN as u64).unwrap();
let mut out = Vec::new();
take(&mut stream, &mut out);
let pace = stream.pace().unwrap();
assert_eq!(pace.held_here(), 0, "the sim's wait completed every read");
let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
let other = Rc::new(Other(RefCell::new(held)));
let weak: Weak<Other> = Rc::downgrade(&other);
let _reg = pace.register(weak);
let waits = p.with_world(|w| w.waits());
let _piece = pace.admit().unwrap();
assert_eq!(p.with_world(|w| w.waits()), waits);
assert_eq!(other.0.borrow().len(), 3);
}
struct Other(RefCell<Vec<Held>>);
impl Collect for Other {
fn collect_one(&self) -> Result<bool, OsError> {
Ok(self.0.borrow_mut().pop().is_some())
}
fn collect_all(&self) -> Result<(), OsError> {
self.0.borrow_mut().clear();
Ok(())
}
fn failed(&self) -> Option<(OsError, u64)> {
None
}
}
#[test]
fn test_a_stream_asking_skips_itself_and_collects_the_other_part() {
let (_p, _s, log, data) = setup(71);
let mut stream = log.read_stream(0..LEN as u64).unwrap();
let pace = stream.pace().unwrap();
let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
let other = Rc::new(Other(RefCell::new(held)));
let weak: Weak<Other> = Rc::downgrade(&other);
let _reg = pace.register(weak);
let mut out = Vec::new();
while out.len() < LEN {
take(&mut stream, &mut out);
}
assert!(stream.next().is_none());
assert!(out == data);
assert!(
other.0.borrow().len() < 4,
"the other part's pieces collected"
);
}
#[test]
fn test_a_paced_stream_asking_skips_itself_and_collects_the_other_part() {
let (_p, s, log, data) = setup(75);
let mut stream = log.read_stream(0..LEN as u64).unwrap();
let pace = stream.pace().unwrap();
let held = (0..2).map(|_| pace.try_charge().unwrap()).collect();
let other = Rc::new(Other(RefCell::new(held)));
let weak: Weak<Other> = Rc::downgrade(&other);
let _reg = pace.register(weak);
let _hold = s.commit_hold();
let mut out = Vec::new();
while out.len() < LEN {
take(&mut stream, &mut out);
}
assert!(out == data);
assert!(other.0.borrow().is_empty());
}
#[test]
fn test_a_failed_collection_ends_the_stream_with_its_error() {
let (p, _s, log, _data) = setup(72);
let mut stream = log.read_stream(0..LEN as u64).unwrap();
let mut out = Vec::new();
holding_the_cap(&p, &mut stream, &mut out);
let pace = stream.pace().unwrap();
p.with_world(|w| {
let next = w.waits() + 1;
w.faults_mut().fail_wait = Some((next, 5));
});
assert!(pace.admit().is_err());
assert_eq!(pace.held_here(), 0, "its charges dropped");
let lost = stream.core.failed().map(|(_, at)| at);
assert_eq!(lost, Some(4 * (out.len() as u64 / 2)));
let waits = p.with_world(|w| w.waits());
let again = pace.try_charge().unwrap();
let more: Vec<_> = (0..3).map(|_| pace.try_charge().unwrap()).collect();
assert!(pace.admit().is_err());
assert_eq!(p.with_world(|w| w.waits()), waits);
drop((again, more));
assert!(matches!(stream.next(), Some(Err(Error::Io { .. }))));
assert!(stream.next().is_none());
let held = (0..4).map(|_| pace.try_charge().unwrap()).collect();
let other = Rc::new(Other(RefCell::new(held)));
let weak: Weak<Other> = Rc::downgrade(&other);
let _reg = pace.register(weak);
let _piece = pace.admit().unwrap();
drop(stream);
assert_eq!(p.with_world(|w| w.waits()), waits, "never waited on again");
}
}
#[cfg(all(test, loom))]
mod loom_tests {
use super::*;
use loom::thread;
#[test]
fn loom_a_give_back_against_a_close_strands_nothing() {
loom::model(|| {
let home = Arc::new(Mutex::new(FreeList::new(vec![1u32])));
lock(&home).lent = 1;
let h = Arc::clone(&home);
let t = thread::spawn(move || lock(&h).give_back(2));
let drained = lock(&home).close();
let back = t.join().unwrap_or(None);
assert!(
lock(&home).bufs.is_empty(),
"a buffer pushed after the close"
);
assert_eq!(drained.len() + usize::from(back.is_some()), 2);
});
}
}