use std::collections::HashMap;
use std::marker::PhantomData;
use std::path::Path;
use std::rc::Rc;
use store_io_buf::{BufPool, PoolConfig};
use store_io_core::errno::is_not_found;
use store_io_core::error::{ByteRange, CorruptionKind, Error, ErrorContext, Named, Op};
use store_io_format::crc32c::{crc32c, crc32c_append};
use store_io_format::exportinfo::{
self, HEADER_LEN, InfoHeader, MAX_LEN, RECORD_LEN, RegionRecord, SPAN,
};
use store_io_platform::{FileMode, FileName, IoOp, Platform, QueueConfig};
use crate::call::{self, CallState};
use crate::exec::{QueueSet, Ran, run};
use crate::export::{ExportHeader, ImportSource, RegionInfo, SinkError};
use crate::file_sink::REGIONS_INFO;
use crate::io::not_written;
use crate::pace::Pace;
use crate::store::StoreOptions;
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct FileSourceOptions {
pub store: StoreOptions,
}
const UNPACED_PIECE: usize = 256 << 10;
const WINDOW: usize = 64 << 10;
const MAX_ITEMS: u32 = crate::v2::MAX_REGION_CAPACITY;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
Fresh,
Begun,
InRegion,
Done,
Failed,
}
fn data_name(index: u32) -> String {
format!("region-{index:05}.data")
}
fn bad(range: Option<(u64, u64)>) -> Error {
Error::Corruption {
kind: CorruptionKind::Metadata,
ctx: ErrorContext {
region: None,
range: range.map(|(start, end)| ByteRange { start, end }),
},
}
}
fn io(op: Op, raw: store_io_core::error::OsError) -> Error {
Error::Io {
op,
raw,
ctx: ErrorContext::default(),
}
}
fn shown(e: &Error) -> SinkError {
match e {
Error::Io { raw, .. } => SinkError::Io(*raw),
_ => SinkError::Refused,
}
}
pub struct FileSource<P: Platform> {
platform: P,
dir: P::Dir,
opts: StoreOptions,
state: State,
call: Option<Rc<CallState>>,
pace: Option<Pace>,
queue: Option<QueueSet<P::Queue>>,
pool: Option<BufPool>,
block: u64,
piece: usize,
info: P::File,
info_len: u64,
header: Option<ExportHeader>,
window: Vec<u8>,
win_at: u64,
pos: u64,
index: u32,
full: HashMap<String, u64>,
run_crc: u32,
crc_at: u64,
want_crc: u32,
file: Option<P::File>,
stream_len: u64,
spans_at: u64,
span: Vec<u8>,
span_index: u64,
span_used: usize,
span_filled: usize,
read_at: u64,
_thread: PhantomData<*const ()>,
}
impl<P: Platform> core::fmt::Debug for FileSource<P> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("FileSource")
.field("state", &self.state)
.field("index", &self.index)
.finish_non_exhaustive()
}
}
impl<P: Platform> FileSource<P> {
pub fn open(platform: P, dir: &Path, opts: FileSourceOptions) -> Result<Self, Error> {
let d = platform.open_dir(dir, false).map_err(|raw| {
if is_not_found(raw) {
Error::NotFound { what: Named::Store }
} else {
io(Op::Open, raw)
}
})?;
let name = FileName::new(REGIONS_INFO).map_err(|_| bad(None))?;
let info = platform
.open_file(&d, &name, FileMode::ReadOnly)
.map_err(|raw| {
if is_not_found(raw) {
bad(None)
} else {
io(Op::Open, raw)
}
})?;
let info_len = platform.size(&info).map_err(|raw| io(Op::Open, raw))?;
Ok(Self {
platform,
dir: d,
opts: opts.store,
state: State::Fresh,
call: None,
pace: None,
queue: None,
pool: None,
block: 0,
piece: 0,
info,
info_len,
header: None,
window: Vec::new(),
win_at: 0,
pos: 0,
index: 0,
full: HashMap::new(),
run_crc: 0,
crc_at: 0,
want_crc: 0,
file: None,
stream_len: 0,
spans_at: 0,
span: Vec::new(),
span_index: 0,
span_used: 0,
span_filled: 0,
read_at: 0,
_thread: PhantomData,
})
}
fn fail(&mut self, e: Error) -> SinkError {
self.state = State::Failed;
let s = shown(&e);
if let Some(c) = &self.call {
c.part_failed(e, self.read_at);
}
s
}
fn read_into(&self, file: &P::File, offset: u64, dst: &mut [u8]) -> Result<usize, Error> {
let (Some(queue), Some(pool)) = (self.queue.as_ref(), self.pool.as_ref()) else {
return Err(bad(None));
};
let block = self.block as usize;
let mut done = 0usize;
while done < dst.len() {
let want = (dst.len() - done).min(self.piece);
let req = want.div_ceil(block) * block;
let mut buf = pool.take(self.piece).map_err(|_| {
not_written(
store_io_core::error::NotWrittenCause::PoolExhausted,
ErrorContext::default(),
)
})?;
if !buf.set_len(req) {
return Err(bad(None));
}
let _held = self.pace.as_ref().map(Pace::admit).transpose()?;
let mut lane = queue.get(0);
let ran = run(
&mut lane,
IoOp::Read {
file,
offset: offset + done as u64,
buf,
},
);
drop(lane);
let (buf, n) = match ran {
Ok(Ran::Done { buf, result }) => (buf, result.map_err(|raw| io(Op::Read, raw))?),
Ok(Ran::Abandoned(raw)) => return Err(io(Op::Read, raw)),
Err((_, raw)) => {
return Err(raw.map_or_else(
|| {
not_written(
store_io_core::error::NotWrittenCause::QueueFull,
ErrorContext::default(),
)
},
|r| io(Op::Read, r),
));
}
};
let Some(buf) = buf else {
return Err(bad(None));
};
let got = n.min(want);
dst[done..done + got].copy_from_slice(&buf.as_slice()[..got]);
done += got;
if n < want {
break;
}
}
Ok(done)
}
fn info_bytes(&mut self, at: u64, n: usize) -> Result<&[u8], Error> {
let end = at.checked_add(n as u64).ok_or_else(|| bad(None))?;
if end > self.info_len {
return Err(bad(Some((at, end))));
}
let have = at >= self.win_at && end <= self.win_at + self.window.len() as u64;
if !have && at < self.crc_at && at != 0 {
return Err(bad(Some((at, self.crc_at))));
}
if !have {
let start = at / self.block * self.block;
let len = (end - start).max(WINDOW as u64).min(self.info_len - start) as usize;
let mut w = std::mem::take(&mut self.window);
w.resize(len, 0);
let got = self.read_into(&self.info, start, &mut w)?;
if got != len {
return Err(bad(Some((start, start + len as u64))));
}
self.window = w;
self.win_at = start;
}
let i = (at - self.win_at) as usize;
if at <= self.crc_at && end > self.crc_at {
let from = (self.crc_at - self.win_at) as usize;
self.run_crc = crc32c_append(self.run_crc, &self.window[from..i + n]);
self.crc_at = end;
}
Ok(&self.window[i..i + n])
}
fn piece_for(&self, block: u64) -> usize {
let piece = self.pace.as_ref().map_or(UNPACED_PIECE, Pace::piece) as u64;
(piece / block * block).max(block).min(SPAN) as usize
}
fn make_pool(&mut self, align: usize) -> Result<(), Error> {
let size = self.piece.next_power_of_two();
let pool =
BufPool::new(&PoolConfig::uniform(align.min(size), size, size, 2)).map_err(|_| {
not_written(
store_io_core::error::NotWrittenCause::PoolExhausted,
ErrorContext::default(),
)
})?;
self.pool = Some(pool);
Ok(())
}
fn header_run(&mut self) -> Result<ExportHeader, Error> {
self.call = call::current();
let ev = self
.platform
.probe(&self.dir, &self.info)
.map_err(|raw| io(Op::Probe, raw))?;
let fs_block = 1u64 << crate::layout::choose_log2_block(&ev, None);
self.block = fs_block;
let qos = crate::store::join_group(&self.platform, &ev, &self.opts, fs_block);
let clock = self.platform.clock();
self.pace = match &self.call {
Some(c) => c.pace_for(&qos, &clock),
None => Some(Pace::new(&qos, clock)),
};
self.piece = self.piece_for(fs_block);
let align = [
4096,
ev.fs.dio_mem_align as usize,
ev.device.logical_block as usize,
]
.into_iter()
.max()
.unwrap_or(4096)
.next_power_of_two();
self.make_pool(align)?;
let queue = self
.platform
.queue(QueueConfig {
depth: 1,
..QueueConfig::default()
})
.map_err(|raw| io(Op::Queue, raw))?;
self.queue = Some(QueueSet::new(vec![queue], 1));
let len = self.info_len;
if len < (HEADER_LEN + 4) as u64 || len > MAX_LEN {
return Err(bad(Some((0, len))));
}
let body = len - 4;
let mut buf = vec![0u8; SPAN as usize];
let mut crc = 0u32;
let mut tail = [0u8; 4];
let mut at = 0u64;
while at < len {
let n = (len - at).min(SPAN) as usize;
let got = self.read_into(&self.info, at, &mut buf[..n])?;
if got != n {
return Err(bad(Some((at, at + n as u64))));
}
let b = body.saturating_sub(at).min(n as u64) as usize;
crc = crc32c_append(crc, &buf[..b]);
for (i, &x) in buf[b..n].iter().enumerate() {
tail[(at + b as u64 - body) as usize + i] = x;
}
at += n as u64;
}
if crc != u32::from_le_bytes(tail) {
return Err(bad(Some((0, len))));
}
self.want_crc = crc;
let h = InfoHeader::decode(self.info_bytes(0, HEADER_LEN)?).map_err(|_| bad(None))?;
let counts = [
h.region_count,
h.omitted_count,
h.lost_count,
h.stale_count,
h.verified_count,
];
if counts.iter().any(|&c| c > MAX_ITEMS) {
return Err(bad(None));
}
let head = exportinfo::head_len(&h).ok_or_else(|| bad(None))?;
if head as u64 > body {
return Err(bad(Some((0, head as u64))));
}
let header = {
let bytes = self.info_bytes(0, head)?;
let (view, used) = exportinfo::decode_prefix(bytes).map_err(|_| bad(None))?;
if used != head {
return Err(bad(None));
}
if view.header != h {
return Err(bad(Some((0, HEADER_LEN as u64))));
}
ExportHeader::from_view(&view)
};
self.block = fs_block.max(header.block_size);
let piece = self.piece_for(self.block);
if piece != self.piece {
self.piece = piece;
self.make_pool(align)?;
}
self.window = Vec::new();
self.win_at = 0;
self.pos = head as u64;
self.span = vec![0u8; SPAN as usize];
self.header = Some(header.clone());
Ok(header)
}
fn next_run(&mut self) -> Result<Option<RegionInfo>, Error> {
let Some(header) = self.header.clone() else {
return Err(bad(None));
};
self.file = None;
let body = self.info_len - 4;
if self.index == header.region_count {
if self.crc_at != body || self.run_crc != self.want_crc {
return Err(bad(Some((0, body))));
}
let all = header
.facts
.verified_up_to
.iter()
.all(|(n, len)| self.full.get(n).is_some_and(|&stream| *len <= stream));
if !all {
return Err(bad(None));
}
return Ok(None);
}
let rec = RegionRecord::decode(self.info_bytes(self.pos, RECORD_LEN)?)
.map_err(|_| bad(Some((self.pos, self.pos + RECORD_LEN as u64))))?;
let spans_at = self.pos + RECORD_LEN as u64;
let next = rec
.spans()
.checked_mul(4)
.and_then(|n| spans_at.checked_add(n))
.filter(|&n| n <= body)
.ok_or_else(|| bad(Some((self.pos, body))))?;
let info = RegionInfo::from_record(&rec, header.block_size);
info.validate(header.block_size)
.map_err(|_| bad(Some((self.pos, spans_at))))?;
let name = FileName::new(&data_name(self.index)).map_err(|_| bad(None))?;
let file = self
.platform
.open_file(&self.dir, &name, FileMode::ReadOnly)
.map_err(|raw| {
if is_not_found(raw) {
bad(None)
} else {
io(Op::Open, raw)
}
})?;
let size = self.platform.size(&file).map_err(|raw| io(Op::Open, raw))?;
if size != rec.stream_len {
return Err(bad(Some((
rec.stream_len.min(size),
rec.stream_len.max(size),
))));
}
if rec.full_copy {
let _was = self.full.insert(info.name.clone(), rec.stream_len);
}
self.pos = next;
self.index += 1;
self.file = Some(file);
self.stream_len = rec.stream_len;
self.spans_at = spans_at;
self.span_index = 0;
self.span_used = 0;
self.span_filled = 0;
self.read_at = 0;
Ok(Some(info))
}
fn read_run(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
if self.read_at == self.stream_len || buf.is_empty() {
return Ok(0);
}
if self.span_used == self.span_filled {
let start = self.span_index * SPAN;
let len = (self.stream_len - start).min(SPAN) as usize;
let mut span = std::mem::take(&mut self.span);
let got = match self.file.as_ref() {
Some(f) => self.read_into(f, start, &mut span[..len]),
None => Err(bad(None)),
};
let got = match got {
Ok(n) => n,
Err(e) => {
self.span = span;
return Err(e);
}
};
let sum = crc32c(&span[..len]);
self.span = span;
if got != len {
return Err(bad(Some((start + got as u64, start + len as u64))));
}
let at = self.spans_at + 4 * self.span_index;
let b = self.info_bytes(at, 4)?;
let want = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
if sum != want {
return Err(bad(Some((start, start + len as u64))));
}
self.span_index += 1;
self.span_filled = len;
self.span_used = 0;
}
let n = buf.len().min(self.span_filled - self.span_used);
buf[..n].copy_from_slice(&self.span[self.span_used..self.span_used + n]);
self.span_used += n;
self.read_at += n as u64;
Ok(n)
}
}
impl<P: Platform> ImportSource for FileSource<P> {
fn header(&mut self) -> Result<ExportHeader, SinkError> {
if self.state != State::Fresh {
return Err(self.fail(bad(None)));
}
match self.header_run() {
Ok(h) => {
self.state = State::Begun;
Ok(h)
}
Err(e) => Err(self.fail(e)),
}
}
fn next_region(&mut self) -> Result<Option<RegionInfo>, SinkError> {
let ready = match self.state {
State::Begun => true,
State::InRegion => self.read_at == self.stream_len,
_ => false,
};
if !ready {
return Err(self.fail(bad(None)));
}
match self.next_run() {
Ok(Some(i)) => {
self.state = State::InRegion;
Ok(Some(i))
}
Ok(None) => {
self.state = State::Done;
Ok(None)
}
Err(e) => Err(self.fail(e)),
}
}
fn read(&mut self, buf: &mut [u8]) -> Result<usize, SinkError> {
if self.state != State::InRegion {
return Err(self.fail(bad(None)));
}
match self.read_run(buf) {
Ok(n) => Ok(n),
Err(e) => Err(self.fail(e)),
}
}
}