use store_io_buf::BufPool;
use store_io_core::errno::{is_end_of_file, is_media_error};
use store_io_core::error::{Error, ErrorContext, NotWrittenCause, Op, OsError};
use store_io_format::pair::{PairReport, decide};
use store_io_format::slot::{
Expect, HeaderFacts, SlotFields, SlotIndex, SlotStatus, encode, header_facts, validate,
};
use store_io_platform::{IoOp, Queue};
use crate::exec::{Lane, Ran, run};
#[derive(Debug, Clone, Copy)]
pub(crate) struct PairAt {
pub(crate) volume: [u8; 16],
pub(crate) object: u64,
pub(crate) a: u64,
pub(crate) b: u64,
pub(crate) log2: u8,
pub(crate) format: u16,
pub(crate) numbered: bool,
}
impl PairAt {
fn expect(&self, i: SlotIndex) -> Expect {
Expect {
volume_uuid: self.volume,
object_id: self.object,
slot_offset: self.offset(i),
slot_index: i,
log2_slot_size: self.log2,
format_major: self.format,
numbered: self.numbered,
}
}
pub(crate) fn offset(&self, i: SlotIndex) -> u64 {
match i {
SlotIndex::A => self.a,
SlotIndex::B => self.b,
}
}
fn size(&self) -> usize {
1usize << self.log2
}
}
#[derive(Debug, Clone)]
pub(crate) struct PairRead {
pub(crate) report: PairReport,
pub(crate) payload: Option<Vec<u8>>,
pub(crate) media_error: bool,
pub(crate) facts: [Option<HeaderFacts>; 2],
pub(crate) twin: bool,
}
fn same(a: Option<&[u8]>, b: Option<&[u8]>) -> bool {
matches!((a, b), (Some(x), Some(y)) if x == y)
}
impl PairRead {
pub(crate) fn after(&self, w: Written) -> PairRead {
let (a, b) = match w.index {
SlotIndex::A => (w.status, self.report.b),
SlotIndex::B => (self.report.a, w.status),
};
let mut facts = self.facts;
facts[usize::from(w.index.as_u8())] = w.facts;
let report = decide(a, b);
let previous_winner = self.report.winner_info().map(|(i, _)| i);
let payload = match report.winner_info() {
Some((i, _)) if i == w.index => w.payload,
Some((i, _)) if Some(i) == previous_winner => self.payload.clone(),
_ => None,
};
PairRead {
report,
payload,
media_error: false,
facts,
twin: false,
}
}
pub(crate) fn possibly_accepted(&self) -> bool {
let other = match self.report.winner_info() {
Some((SlotIndex::A, _)) => self.report.b,
Some((SlotIndex::B, _)) => self.report.a,
None => return false,
};
self.twin || !matches!(other, SlotStatus::Valid(_) | SlotStatus::Blank)
}
pub(crate) fn blank() -> PairRead {
PairRead {
report: decide(SlotStatus::Blank, SlotStatus::Blank),
payload: None,
media_error: false,
facts: [None, None],
twin: false,
}
}
}
#[derive(Debug, Clone)]
pub(crate) struct Written {
pub(crate) index: SlotIndex,
pub(crate) status: SlotStatus,
pub(crate) payload: Option<Vec<u8>>,
pub(crate) facts: Option<HeaderFacts>,
}
fn pool_err(ctx: ErrorContext) -> Error {
Error::NotWritten {
cause: NotWrittenCause::PoolExhausted,
ctx,
}
}
pub(crate) fn unreadable(raw: OsError) -> bool {
is_media_error(raw) || is_end_of_file(raw)
}
struct OneSlot {
status: SlotStatus,
payload: Option<Vec<u8>>,
media: bool,
facts: Option<HeaderFacts>,
}
impl OneSlot {
fn unreadable(media: bool) -> Self {
Self {
status: SlotStatus::ReadError,
payload: None,
media,
facts: None,
}
}
}
fn read_one<Q: Queue>(
q: &mut Lane<Q>,
file: &Q::File,
pool: &BufPool,
at: &PairAt,
i: SlotIndex,
) -> Result<OneSlot, Error> {
let buf = pool
.take(at.size())
.map_err(|_| pool_err(ErrorContext::default()))?;
let offset = at.offset(i);
let ran = run(q, IoOp::Read { file, offset, buf }).map_err(|(_b, raw)| match raw {
Some(raw) => Error::Io {
op: Op::Read,
raw,
ctx: ErrorContext::default(),
},
None => pool_err(ErrorContext::default()),
})?;
let fail = |raw| Error::Io {
op: Op::Read,
raw,
ctx: ErrorContext::default(),
};
let (got, result) = match ran {
Ran::Done { buf, result } => (buf, result),
Ran::Abandoned(raw) => return Err(fail(raw)),
};
finish_one(at, i, got, result)
}
pub(crate) fn read_pair<Q: Queue>(
q: &mut Lane<Q>,
file: &Q::File,
pool: &BufPool,
at: &PairAt,
) -> Result<PairRead, Error> {
let a = read_one(q, file, pool, at, SlotIndex::A)?;
let b = read_one(q, file, pool, at, SlotIndex::B)?;
Ok(assemble(a, b))
}
fn landed(
got: Option<store_io_buf::IoBuf>,
result: Result<usize, OsError>,
len: usize,
) -> Result<Result<store_io_buf::IoBuf, bool>, Error> {
let media = matches!(result, Err(raw) if is_media_error(raw));
match result {
Err(raw) if !unreadable(raw) => Err(Error::Io {
op: Op::Read,
raw,
ctx: ErrorContext::default(),
}),
Ok(n) if n == len => Ok(got.ok_or(media)),
_ => Ok(Err(media)),
}
}
fn judge_slot(at: &PairAt, i: SlotIndex, bytes: &[u8]) -> OneSlot {
let expect = at.expect(i);
let status = validate(bytes, &expect);
let payload = match status {
SlotStatus::Valid(info) => Some(info.payload(bytes).to_vec()),
_ => None,
};
OneSlot {
status,
payload,
media: false,
facts: header_facts(bytes, &expect),
}
}
fn finish_one(
at: &PairAt,
i: SlotIndex,
got: Option<store_io_buf::IoBuf>,
result: Result<usize, OsError>,
) -> Result<OneSlot, Error> {
Ok(match landed(got, result, at.size())? {
Ok(buf) => judge_slot(at, i, buf.as_slice()),
Err(media) => OneSlot::unreadable(media),
})
}
fn assemble(a: OneSlot, b: OneSlot) -> PairRead {
let report = decide(a.status, b.status);
let twin = same(a.payload.as_deref(), b.payload.as_deref());
let payload = match report.winner_info() {
Some((SlotIndex::A, _)) => a.payload,
Some((SlotIndex::B, _)) => b.payload,
None => None,
};
PairRead {
report,
payload,
media_error: a.media || b.media,
facts: [a.facts, b.facts],
twin,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Part {
One(SlotIndex),
Both,
}
pub(crate) fn read_pairs<Q: Queue>(
q: &mut Lane<Q>,
file: &Q::File,
pool: &BufPool,
ats: &[PairAt],
depth: usize,
) -> Result<Vec<PairRead>, Error> {
let io = |raw| Error::Io {
op: Op::Read,
raw,
ctx: ErrorContext::default(),
};
crate::exec::clear_stale(q).map_err(io)?;
let mut reads: Vec<(usize, Part)> = Vec::with_capacity(ats.len());
for (i, at) in ats.iter().enumerate() {
if at.a.checked_add(at.size() as u64) == Some(at.b) {
reads.push((i, Part::Both));
} else {
reads.push((i, Part::One(SlotIndex::A)));
reads.push((i, Part::One(SlotIndex::B)));
}
}
let mut slots: Vec<[Option<OneSlot>; 2]> = (0..ats.len()).map(|_| [None, None]).collect();
let (mut next, mut arrived, mut in_flight) = (0usize, 0usize, 0usize);
let own_class = |len: usize| pool.class_len(len).unwrap_or(len);
let depth = depth.max(1);
while arrived < reads.len() {
while next < reads.len() && in_flight < depth {
let (i, part) = reads[next];
let (offset, len) = match part {
Part::One(s) => (ats[i].offset(s), ats[i].size()),
Part::Both => (ats[i].a, 2 * ats[i].size()),
};
let taken = if in_flight > 0 {
pool.take_within(len, own_class(len))
} else {
pool.take(len)
};
let Ok(buf) = taken else {
if in_flight > 0 {
break;
}
return Err(pool_err(ErrorContext::default()));
};
match q.submit(IoOp::Read { file, offset, buf }, next as u64) {
Ok(()) => {
next += 1;
in_flight += 1;
}
Err(r) if r.raw.is_none() && in_flight > 0 => break,
Err(r) => {
return Err(match r.raw {
Some(raw) => io(raw),
None => pool_err(ErrorContext::default()),
});
}
}
}
let _arrived = q.wait(1).map_err(io)?;
for c in q.done.drain() {
debug_assert!(
c.buf.is_some() || c.result.is_err(),
"a read succeeded without its buffer (platform contract 1)"
);
in_flight -= 1;
arrived += 1;
let (i, part) = reads[c.tag as usize];
let at = &ats[i];
match part {
Part::One(s) => {
slots[i][usize::from(s.as_u8())] = Some(finish_one(at, s, c.buf, c.result)?);
}
Part::Both => match landed(c.buf, c.result, 2 * at.size())? {
Ok(buf) => {
let (a, b) = buf.as_slice().split_at(at.size());
slots[i] = [
Some(judge_slot(at, SlotIndex::A, a)),
Some(judge_slot(at, SlotIndex::B, b)),
];
}
Err(media) => {
slots[i] = [
Some(OneSlot::unreadable(media)),
Some(OneSlot::unreadable(media)),
];
}
},
}
}
}
debug_assert!(
slots.iter().all(|s| s.iter().all(Option::is_some)),
"a slot read never arrived"
);
Ok(slots
.into_iter()
.map(|[a, b]| {
assemble(
a.unwrap_or_else(|| OneSlot::unreadable(false)),
b.unwrap_or_else(|| OneSlot::unreadable(false)),
)
})
.collect())
}
pub(crate) fn reread_failed<Q: Queue>(
q: &mut Lane<Q>,
file: &Q::File,
pool: &BufPool,
at: &PairAt,
first: PairRead,
) -> Result<(PairRead, bool), Error> {
let failed = |s: SlotStatus| s == SlotStatus::ReadError;
if !failed(first.report.a) && !failed(first.report.b) {
return Ok((first, false));
}
let mut status = [first.report.a, first.report.b];
let mut facts = first.facts;
let mut payloads: [Option<Vec<u8>>; 2] = [None, None];
let mut media = false;
for (i, slot) in [SlotIndex::A, SlotIndex::B].into_iter().enumerate() {
if failed(status[i]) {
let one = read_one(q, file, pool, at, slot)?;
status[i] = one.status;
facts[i] = one.facts;
payloads[i] = one.payload;
media |= one.media;
}
}
let report = decide(status[0], status[1]);
let both = status.iter().all(|s| matches!(s, SlotStatus::Valid(_)));
let reread = payloads.iter().find_map(|p| p.as_deref());
let twin = both
&& match (first.report.a, first.report.b) {
(SlotStatus::ReadError, SlotStatus::ReadError) => {
same(payloads[0].as_deref(), payloads[1].as_deref())
}
_ => same(first.payload.as_deref(), reread),
};
let payload = match report.winner_info() {
Some((slot, _)) => {
let i = usize::from(slot.as_u8());
if failed([first.report.a, first.report.b][i]) {
payloads[i].take()
} else {
first.payload
}
}
None => None,
};
Ok((
PairRead {
report,
payload,
media_error: media,
facts,
twin,
},
true,
))
}
pub(crate) struct NextSlot<'a> {
pub(crate) index: SlotIndex,
pub(crate) generation: u64,
pub(crate) owner_generation: u64,
pub(crate) prev_header_crc: u32,
pub(crate) payload: &'a [u8],
pub(crate) meta_seq: u64,
pub(crate) compat_flags: u32,
}
pub(crate) fn write_slot<Q: Queue>(
q: &mut Lane<Q>,
file: &Q::File,
pool: &BufPool,
at: &PairAt,
next: &NextSlot<'_>,
dsync: bool,
) -> Result<Written, Error> {
let ctx = ErrorContext::default();
debug_assert_eq!(
next.meta_seq != 0,
at.format == store_io_format::slot::FORMAT_2 && at.numbered,
"meta_seq for a numbered format-2 object only"
);
let mut buf = pool.take(at.size()).map_err(|_| pool_err(ctx))?;
let fields = SlotFields {
volume_uuid: at.volume,
object_id: at.object,
generation: next.generation,
owner_generation: next.owner_generation,
slot_offset: at.offset(next.index),
slot_index: next.index,
log2_slot_size: at.log2,
prev_header_crc32c: next.prev_header_crc,
format_major: at.format,
meta_seq: next.meta_seq,
compat_flags: next.compat_flags,
};
let crc = encode(buf.as_mut_slice(), &fields, next.payload).map_err(|e| match e {
store_io_format::slot::EncodeError::Format => Error::Unsupported {
what: store_io_core::error::Capability::FormatVersion,
},
_ => Error::NotWritten {
cause: NotWrittenCause::TooLarge,
ctx,
},
})?;
let offset = at.offset(next.index);
let expect = at.expect(next.index);
let status = validate(buf.as_slice(), &expect);
let SlotStatus::Valid(info) = status else {
return Err(Error::Corruption {
kind: store_io_core::error::CorruptionKind::Metadata,
ctx,
});
};
if info.header_crc32c != crc {
return Err(Error::Corruption {
kind: store_io_core::error::CorruptionKind::Metadata,
ctx,
});
}
let written = Written {
index: next.index,
status,
payload: Some(info.payload(buf.as_slice()).to_vec()),
facts: header_facts(buf.as_slice(), &expect),
};
let ran = run(
q,
IoOp::Write {
file,
offset,
buf,
dsync,
},
)
.map_err(|(_b, raw)| Error::NotWritten {
cause: if raw.is_some() {
NotWrittenCause::Interrupted
} else {
NotWrittenCause::QueueFull
},
ctx,
})?;
let result = match ran {
Ran::Done { result, .. } => result,
Ran::Abandoned(raw) => Err(raw),
};
match result {
Ok(n) if n == at.size() => Ok(written),
Ok(_) => Err(Error::DurabilityUnknown {
op: Op::Write,
raw: None,
ctx,
}),
Err(raw) => Err(Error::DurabilityUnknown {
op: Op::Write,
raw: Some(raw),
ctx,
}),
}
}
pub(crate) fn plan_next<'a>(
read: &PairRead,
payload: &'a [u8],
owner_generation: u64,
) -> Result<NextSlot<'a>, Error> {
let fork = || Error::Corruption {
kind: store_io_core::error::CorruptionKind::Fork,
ctx: ErrorContext::default(),
};
let index = read.report.next_write().ok_or_else(fork)?;
let (generation, prev, compat_flags) = match read.report.winner_info() {
Some((_, info)) if !info.writable() => {
return Err(Error::Unsupported {
what: store_io_core::error::Capability::FormatVersion,
});
}
Some((_, info)) => (
info.generation.checked_add(1).ok_or_else(fork)?,
info.header_crc32c,
info.compat_flags,
),
None => (1, 0, 0),
};
Ok(NextSlot {
index,
generation,
owner_generation,
prev_header_crc: prev,
payload,
meta_seq: 0,
compat_flags,
})
}
#[cfg(all(test, not(loom)))]
mod tests {
#![allow(clippy::unwrap_used)]
use std::path::Path;
use store_io_sim::{SimConfig, SimPlatform};
use super::*;
use crate::store::{Store, StoreOptions, pair_volume};
#[test]
fn test_after_a_write_matches_a_fresh_read() {
let p = SimPlatform::new(SimConfig::volatile(41));
let opts = StoreOptions {
container_format: store_io_format::slot::FORMAT_1,
..StoreOptions::default()
};
let s = Store::create(p, Path::new("/db"), opts).unwrap();
let inner = &s.inner;
let at = pair_volume(inner.volume, &inner.layout);
let mut prev = {
let mut q = inner.queues.get(0);
read_pair(&mut q, &inner.file, &inner.pool, &at).unwrap()
};
let payload = prev.payload.clone().unwrap();
for _ in 0..2 {
let mut q = inner.queues.get(0);
let next = plan_next(&prev, &payload, 1).unwrap();
let w = write_slot(&mut q, &inner.file, &inner.pool, &at, &next, false).unwrap();
let after = prev.after(w);
let read = read_pair(&mut q, &inner.file, &inner.pool, &at).unwrap();
assert_eq!(after.report.a, read.report.a);
assert_eq!(after.report.b, read.report.b);
assert_eq!(after.facts, read.facts, "header facts, A then B");
assert!(after.facts.iter().all(Option::is_some));
prev = read;
}
}
}