use std::ops::ControlFlow;
use store_io_core::errno::is_media_error;
use store_io_core::error::{Error, ErrorContext, NotWrittenCause, Op, OsError};
use store_io_core::observe::Counter;
use store_io_platform::{IoBuf, IoOp, Platform, Queue, RawResult};
use crate::exec::{Lane, LaneGuard, Ran, run};
use crate::io::{ctx_range, not_written};
use crate::pace::Pace;
use crate::store::{LiveRegion, Store};
const DEPTH: usize = 8;
#[derive(Debug, PartialEq, Eq)]
pub enum ScanItem<'a> {
Data {
offset: u64,
bytes: &'a [u8],
},
Unreadable {
start: u64,
end: u64,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct ScanSummary {
pub end: u64,
pub unreadable: u64,
pub stopped: bool,
}
struct Emitter<F> {
visit: F,
bad: Option<(u64, u64)>,
sum: ScanSummary,
held: Option<Vec<Held>>,
}
enum Held {
Data(u64, Vec<u8>),
Unreadable(u64, u64),
}
impl<F: FnMut(ScanItem<'_>, &mut dyn FnMut() -> bool) -> ControlFlow<()>> Emitter<F> {
fn hold(&mut self) {
let _held = self.held.get_or_insert_with(Vec::new);
}
fn release(&mut self) {
for item in self.held.take().unwrap_or_default() {
match item {
Held::Data(offset, bytes) => self.data(offset, &bytes, &mut || true),
Held::Unreadable(start, end) => self.unreadable(start, end, &mut || true),
}
}
}
fn flush_bad(&mut self, drain: &mut dyn FnMut() -> bool) {
if let Some((start, end)) = self.bad.take() {
if (self.visit)(ScanItem::Unreadable { start, end }, drain).is_break() {
self.sum.stopped = true;
}
}
}
fn data(&mut self, offset: u64, bytes: &[u8], drain: &mut dyn FnMut() -> bool) {
if let Some(held) = &mut self.held {
held.push(Held::Data(offset, bytes.to_vec()));
return;
}
if self.sum.stopped {
return;
}
self.flush_bad(drain);
if self.sum.stopped || bytes.is_empty() {
return;
}
self.sum.end = offset + bytes.len() as u64;
if (self.visit)(ScanItem::Data { offset, bytes }, drain).is_break() {
self.sum.stopped = true;
}
}
fn unreadable(&mut self, start: u64, end: u64, drain: &mut dyn FnMut() -> bool) {
if let Some(held) = &mut self.held {
held.push(Held::Unreadable(start, end));
return;
}
if self.sum.stopped || start >= end {
return;
}
self.sum.unreadable += end - start;
self.sum.end = end;
match &mut self.bad {
Some((_, e)) if *e == start => *e = end,
_ => {
self.flush_bad(drain);
self.bad = Some((start, end));
}
}
}
fn finish(mut self) -> ScanSummary {
if !self.sum.stopped {
self.flush_bad(&mut || true);
}
self.sum
}
}
struct Pending {
start: u64,
len: usize,
done: Option<(RawResult<usize>, Option<IoBuf>)>,
charge: Option<crate::pace::Held>,
}
fn read_error(raw: OsError, c: ErrorContext) -> Error {
Error::Io {
op: Op::Read,
raw,
ctx: c,
}
}
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub(crate) fn scan_reported<F>(
&self,
r: &LiveRegion,
from: u64,
to: u64,
mut visit: F,
) -> Result<ScanSummary, Error>
where
F: FnMut(ScanItem<'_>) -> ControlFlow<()>,
{
let obs = &self.inner.observer;
let pace = self.inner.domain.new_pace();
self.scan_region(r, from, to, true, pace.as_ref(), |item, _drain| {
if matches!(item, ScanItem::Unreadable { .. }) {
obs.add(Counter::MediaErrors, 1);
}
visit(item)
})
}
pub(crate) fn scan_region<F>(
&self,
r: &LiveRegion,
from: u64,
to: u64,
calls_store: bool,
pace: Option<&Pace>,
visit: F,
) -> Result<ScanSummary, Error>
where
F: FnMut(ScanItem<'_>, &mut dyn FnMut() -> bool) -> ControlFlow<()>,
{
let block = self.block();
let to = to.min(r.map.len());
let c = ctx_range(r, from, to);
if from % block != 0 {
return Err(not_written(NotWrittenCause::Misaligned, c));
}
if from > to {
return Err(not_written(NotWrittenCause::OutOfBounds, c));
}
let mut out = Emitter {
visit,
bad: None,
held: None,
sum: ScanSummary {
end: from,
..ScanSummary::default()
},
};
if from == to {
return Ok(out.finish());
}
let chunk = pace.map_or(self.chunk(), |p| p.piece().min(self.chunk()));
let class = self.inner.pool.class_len(chunk).unwrap_or(chunk);
let mut free: Vec<IoBuf> = (0..DEPTH)
.map_while(|_| match pace {
Some(_) => self.inner.pool.take_capped(chunk, class).ok(),
None => self.inner.pool.take_within(chunk, chunk).ok(),
})
.collect();
if free.is_empty() {
return Err(not_written(NotWrittenCause::PoolExhausted, c));
}
let depth = free.len();
let mut ring: Vec<Option<Pending>> = (0..depth).map(|_| None).collect();
let several = to - from > chunk as u64 || r.map.run_end(from) < to;
let take = || {
if several && depth > 1 {
self.par_lane(r, Op::Read)
} else {
self.lane(r, Op::Read)
}
};
let first = take()?;
let mut deferred = calls_store && first.is_shared();
let mut lane: Option<LaneGuard<'_, P::Queue>> = Some(first);
let file = &self.inner.file;
let (mut submitted, mut delivered) = (0u64, 0u64);
let mut next = from;
let mut in_flight = 0usize;
let mut failure: Option<Error> = None;
let mut drained: Option<OsError> = None;
loop {
let progress = (submitted, delivered);
while failure.is_none() && !out.sum.stopped && next < to && !free.is_empty() {
if lane.is_none() {
match take() {
Ok(l) => {
deferred = calls_store && l.is_shared();
lane = Some(l);
}
Err(e) => {
failure = Some(e);
break;
}
}
}
let Some(lane) = &mut lane else { break };
let Some(mut buf) = free.pop() else { break };
let len = (to - next).min(chunk as u64).min(r.map.run_left(next)) as usize;
let charge = match pace {
None => None,
Some(p) => match p.try_charge() {
Some(ch) => Some(ch),
None if in_flight > 0 => {
free.push(buf);
break;
}
None => match p.wait_charge() {
Some(ch) => Some(ch),
None => {
free.push(buf);
failure = Some(not_written(NotWrittenCause::Interrupted, c));
break;
}
},
},
};
let _fits = buf.set_len(len);
let op = IoOp::Read {
file,
offset: r.map.locate(next, len as u64),
buf,
};
match lane.submit(op, submitted) {
Ok(()) => {
ring[(submitted % depth as u64) as usize] = Some(Pending {
start: next,
len,
done: None,
charge,
});
submitted += 1;
next += len as u64;
in_flight += 1;
}
Err(rejected) => {
if let Some(b) = rejected.buf {
free.push(b);
}
match rejected.raw {
None if in_flight > 0 => break,
None => failure = Some(not_written(NotWrittenCause::QueueFull, c)),
Some(raw) => failure = Some(read_error(raw, c)),
}
}
}
}
if deferred {
if let Some(l) = &mut lane {
if let Err(e) = collect(l, &mut ring, depth, &mut in_flight) {
return Err(read_error(e, c));
}
}
lane = None;
}
while delivered < submitted {
let idx = (delivered % depth as u64) as usize;
if ring[idx].as_ref().is_none_or(|p| p.done.is_none()) {
break;
}
let Some(Pending {
start,
len,
done: Some((result, buf)),
..
}) = ring[idx].take()
else {
break;
};
delivered += 1;
let mut buf = buf;
debug_assert!(
buf.is_some() || result.is_err(),
"a read succeeded without its buffer (platform contract 1)"
);
if let (Ok(n), Some(_)) = (&result, &buf) {
self.inner.observer.add(Counter::Reads, 1);
self.inner
.observer
.add(Counter::ReadBytes, (*n).min(len) as u64);
}
if failure.is_none() && !out.sum.stopped {
match result {
Ok(n) => match &buf {
Some(b) => {
let n = n.min(len);
let mut drain = drainer(
&mut lane,
&mut ring,
depth,
&mut in_flight,
&mut drained,
);
out.data(start, &b.as_slice()[..n], &mut drain);
out.unreadable(start + n as u64, start + len as u64, &mut drain);
drop(drain);
if let Some(e) = drained.take() {
return Err(read_error(e, c));
}
}
None => {
failure = Some(not_written(NotWrittenCause::Interrupted, c));
}
},
Err(raw) if is_media_error(raw) => {
if let Some(l) = &mut lane {
if let Err(e) = collect(l, &mut ring, depth, &mut in_flight) {
return Err(read_error(e, c));
}
}
match buf.take() {
Some(b) if !deferred => {
let bisected = match &mut lane {
Some(l) => self
.bisect(l, r, start, len as u64, b, &mut out, c, pace),
None => Err(not_written(NotWrittenCause::Interrupted, c)),
};
match bisected {
Ok(b) => buf = Some(b),
Err(e) => failure = Some(e),
}
}
Some(b) => match self.lane(r, Op::Read) {
Ok(mut l) => {
out.hold();
let bisected = self.bisect(
&mut l, r, start, len as u64, b, &mut out, c, pace,
);
drop(l);
out.release();
match bisected {
Ok(b) => buf = Some(b),
Err(_) if out.sum.stopped => {}
Err(e) => failure = Some(e),
}
}
Err(e) => failure = Some(e),
},
None => {
failure = Some(not_written(NotWrittenCause::Interrupted, c));
}
}
}
Err(raw) => failure = Some(read_error(raw, c)),
}
}
if let Some(b) = buf {
free.push(b);
}
}
if in_flight == 0 && (next >= to || out.sum.stopped || failure.is_some()) {
break;
}
if in_flight == 0 && (submitted, delivered) == progress {
return Err(not_written(NotWrittenCause::Interrupted, c));
}
if in_flight > 0 {
let Some(l) = &mut lane else {
return Err(not_written(NotWrittenCause::Interrupted, c));
};
if let Err(e) = collect_one(l, &mut ring, depth, &mut in_flight) {
return Err(read_error(e, c));
}
}
}
match failure {
Some(e) => Err(e),
None => Ok(out.finish()),
}
}
#[allow(clippy::too_many_arguments)]
fn bisect<F>(
&self,
lane: &mut Lane<P::Queue>,
r: &LiveRegion,
start: u64,
len: u64,
mut buf: IoBuf,
out: &mut Emitter<F>,
c: ErrorContext,
pace: Option<&Pace>,
) -> Result<IoBuf, Error>
where
F: FnMut(ScanItem<'_>, &mut dyn FnMut() -> bool) -> ControlFlow<()>,
{
if out.sum.stopped {
return Ok(buf);
}
let sector = self.sector();
let _fits = buf.set_len(len as usize);
let piece = pace.map(Pace::admit).transpose()?;
let done = run(
lane,
IoOp::Read {
file: &self.inner.file,
offset: r.map.locate(start, len),
buf,
},
);
drop(piece);
let (result, back) = match done {
Ok(Ran::Done { buf, result }) => (result, buf),
Ok(Ran::Abandoned(raw)) => return Err(read_error(raw, c)),
Err((_refused, Some(raw))) => return Err(read_error(raw, c)),
Err((_refused, None)) => return Err(not_written(NotWrittenCause::QueueFull, c)),
};
let Some(buf) = back else {
return Err(match result {
Err(raw) => read_error(raw, c),
Ok(_) => not_written(NotWrittenCause::Interrupted, c),
});
};
match result {
Ok(n) => {
let n = (n as u64).min(len);
self.inner.observer.add(Counter::Reads, 1);
self.inner.observer.add(Counter::ReadBytes, n);
out.data(start, &buf.as_slice()[..n as usize], &mut || true);
out.unreadable(start + n, start + len, &mut || true);
Ok(buf)
}
Err(raw) if is_media_error(raw) => {
if len <= sector {
out.unreadable(start, start + len, &mut || true);
return Ok(buf);
}
let block = self.block();
let unit = if len > block { block } else { sector };
let half = (len / unit).div_ceil(2) * unit;
let buf = self.bisect(lane, r, start, half, buf, out, c, pace)?;
self.bisect(lane, r, start + half, len - half, buf, out, c, pace)
}
Err(_) if len < self.block() => {
out.unreadable(start, start + len, &mut || true);
Ok(buf)
}
Err(raw) => Err(read_error(raw, c)),
}
}
}
fn collect_one<Q: Queue>(
lane: &mut Lane<Q>,
ring: &mut [Option<Pending>],
depth: usize,
in_flight: &mut usize,
) -> Result<(), OsError> {
let _added = lane.wait(1)?;
for comp in lane.done.drain() {
*in_flight = in_flight.saturating_sub(1);
if let Some(p) = ring[(comp.tag % depth as u64) as usize].as_mut() {
p.done = Some((comp.result, comp.buf));
p.charge = None;
}
}
Ok(())
}
fn drainer<'x, Q: Queue>(
lane: &'x mut Option<LaneGuard<'_, Q>>,
ring: &'x mut [Option<Pending>],
depth: usize,
in_flight: &'x mut usize,
err: &'x mut Option<OsError>,
) -> impl FnMut() -> bool + 'x {
move || {
if err.is_none() {
if let Some(l) = lane.as_mut() {
if let Err(e) = collect(l, &mut *ring, depth, &mut *in_flight) {
*err = Some(e);
}
}
}
*in_flight == 0
}
}
fn collect<Q: Queue>(
lane: &mut Lane<Q>,
ring: &mut [Option<Pending>],
depth: usize,
in_flight: &mut usize,
) -> Result<(), OsError> {
while *in_flight > 0 {
collect_one(lane, ring, depth, in_flight)?;
}
Ok(())
}