use std::sync::Arc;
use std::sync::atomic::Ordering;
use store_io_core::error::{Error, ErrorContext, NotWrittenCause, Op, OsError};
use store_io_core::id::RegionId;
use store_io_format::exportinfo::{self, InfoHeader, RECORD_LEN, RegionRecord};
use store_io_format::meta::{FillPattern, RegionKind, RegionName};
use store_io_format::meta2::max_object;
use store_io_platform::Platform;
use crate::call::{self, CallState};
use crate::copy::{CopyPlan, Facts};
use crate::io::{ctx_range, not_written};
use crate::lifecycle::retired_error;
use crate::slot_handle::committed_object;
use crate::store::{LiveRegion, Store, StoreOptions};
use crate::stream::{Chunk, Locator, ReadStream};
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct ExportOptions {
pub regions: Option<Vec<String>>,
pub depth: Option<usize>,
pub accept_stale_pages: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct VolumeFacts {
pub omitted: Vec<RegionId>,
pub lost_pages: Vec<u32>,
pub stale_pages: Vec<u32>,
pub verified_up_to: Vec<(String, u64)>,
pub superseded: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ExportHeader {
pub region_count: u32,
pub block_size: u64,
pub facts: VolumeFacts,
pub writable: bool,
pub lend: usize,
pub total: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct RegionInfo {
pub name: String,
pub kind: RegionKind,
pub generation: u64,
pub tag: u32,
pub cap: Option<u64>,
pub fill: FillPattern,
pub fill_key: [u8; 16],
pub block_size: u64,
pub data_size: u64,
pub durable_len: Option<u64>,
pub object_len: Option<u64>,
pub full_copy: bool,
pub superseded: bool,
}
impl RegionInfo {
#[must_use]
pub fn stream_len(&self) -> u64 {
match self.kind {
RegionKind::Slot => self.object_len.unwrap_or(0),
_ => self.durable_len.unwrap_or(self.data_size),
}
}
pub fn validate(&self, block_size: u64) -> Result<(), Error> {
let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
if RegionName::new(&self.name).is_err() || self.generation == 0 {
return Err(bad());
}
let block = self.block_size;
if block != block_size || !block.is_power_of_two() || !(512..=1 << 16).contains(&block) {
return Err(bad());
}
if self.data_size % block != 0 {
return Err(bad());
}
if let Some(cap) = self.cap {
if cap < self.data_size || cap % block != 0 {
return Err(bad());
}
}
let slot = self.kind == RegionKind::Slot;
if let Some(d) = self.durable_len {
if self.kind != RegionKind::Append || self.full_copy || d > self.data_size {
return Err(bad());
}
}
if let Some(n) = self.object_len {
let most = max_object(block.trailing_zeros() as u8) as u64;
if !slot || n > most {
return Err(bad());
}
}
if self.full_copy && slot {
return Err(bad());
}
let footprint = block
.checked_mul(2)
.and_then(|h| h.checked_add(self.cap.unwrap_or(0).max(self.data_size)));
if footprint.is_none() {
return Err(bad());
}
Ok(())
}
pub(crate) fn to_record(&self) -> Option<RegionRecord> {
Some(RegionRecord {
name: RegionName::new(&self.name).ok()?,
kind: self.kind,
fill: self.fill,
fill_key: self.fill_key,
tag: self.tag,
generation: self.generation,
cap: self.cap,
data_size: self.data_size,
durable_len: self.durable_len,
object_len: match self.object_len {
Some(n) => Some(u32::try_from(n).ok()?),
None => None,
},
stream_len: self.stream_len(),
full_copy: self.full_copy,
superseded: self.superseded,
})
}
pub(crate) fn from_record(rec: &RegionRecord, block_size: u64) -> Self {
Self {
name: rec.name.as_str().to_owned(),
kind: rec.kind,
generation: rec.generation,
tag: rec.tag,
cap: rec.cap,
fill: rec.fill,
fill_key: rec.fill_key,
block_size,
data_size: rec.data_size,
durable_len: rec.durable_len,
object_len: rec.object_len.map(u64::from),
full_copy: rec.full_copy,
superseded: rec.superseded,
}
}
pub fn to_bytes(&self) -> Result<[u8; RECORD_LEN], Error> {
let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
let rec = self.to_record().ok_or_else(bad)?;
rec.check().map_err(|_| bad())?;
Ok(rec.encode())
}
pub fn from_bytes(bytes: &[u8], block_size: u64) -> Result<Self, Error> {
let rec = RegionRecord::decode(bytes)
.map_err(|_| not_written(NotWrittenCause::InvalidOptions, ErrorContext::default()))?;
let info = Self::from_record(&rec, block_size);
info.validate(block_size)?;
Ok(info)
}
}
impl ExportHeader {
pub fn to_bytes(&self) -> Result<Vec<u8>, Error> {
let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
let block = self.block_size;
if !block.is_power_of_two() {
return Err(bad());
}
let mut omitted: Vec<u64> = self.facts.omitted.iter().map(|id| id.get()).collect();
omitted.sort_unstable();
let mut lost = self.facts.lost_pages.clone();
lost.sort_unstable();
let mut stale = self.facts.stale_pages.clone();
stale.sort_unstable();
let mut verified = Vec::with_capacity(self.facts.verified_up_to.len());
for (name, len) in &self.facts.verified_up_to {
verified.push((RegionName::new(name).map_err(|_| bad())?, *len));
}
verified.sort_by_key(|(n, _)| n.as_padded());
let h = InfoHeader {
writable: self.writable,
block_log2: block.trailing_zeros() as u8,
region_count: self.region_count,
lend: u32::try_from(self.lend).unwrap_or(u32::MAX),
total: self.total,
omitted_count: 0,
lost_count: 0,
stale_count: 0,
verified_count: 0,
};
let counted = InfoHeader {
omitted_count: u32::try_from(omitted.len()).map_err(|_| bad())?,
lost_count: u32::try_from(lost.len()).map_err(|_| bad())?,
stale_count: u32::try_from(stale.len()).map_err(|_| bad())?,
verified_count: u32::try_from(verified.len()).map_err(|_| bad())?,
..h
};
let len = exportinfo::head_len(&counted)
.and_then(|n| n.checked_add(4))
.ok_or_else(bad)?;
let mut out = vec![0u8; len];
let n = exportinfo::encode_head(&h, &omitted, &lost, &stale, &verified, &mut out)
.map_err(|_| bad())?;
out.truncate(n);
Ok(out)
}
pub fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {
let info = exportinfo::decode_head(bytes)
.map_err(|_| not_written(NotWrittenCause::InvalidOptions, ErrorContext::default()))?;
Ok(Self::from_view(&info))
}
pub(crate) fn from_view(info: &exportinfo::ExportInfo<'_>) -> Self {
let h = info.header;
Self {
region_count: h.region_count,
block_size: 1u64 << h.block_log2,
facts: VolumeFacts {
omitted: info.omitted().map(RegionId::from_raw).collect(),
lost_pages: info.lost_pages().collect(),
stale_pages: info.stale_pages().collect(),
verified_up_to: info
.verified()
.map(|(n, len)| (n.as_str().to_owned(), len))
.collect(),
superseded: Vec::new(),
},
writable: h.writable,
lend: h.lend as usize,
total: h.total,
}
}
}
pub trait ImportSource {
fn header(&mut self) -> Result<ExportHeader, SinkError>;
fn next_region(&mut self) -> Result<Option<RegionInfo>, SinkError>;
fn read(&mut self, buf: &mut [u8]) -> Result<usize, SinkError>;
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct ExportReport {
pub regions: usize,
pub bytes: u64,
pub region: Option<String>,
pub offset: u64,
pub writable: bool,
pub complete: bool,
pub facts: VolumeFacts,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum SinkError {
Io(OsError),
Refused,
Other(u32),
}
impl core::fmt::Display for SinkError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::Io(raw) => write!(f, "sink I/O error {}", raw.code),
Self::Refused => f.write_str("sink refused the call"),
Self::Other(code) => write!(f, "sink error {code}"),
}
}
}
impl std::error::Error for SinkError {}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct ExportError {
pub error: Error,
pub sink: Option<SinkError>,
pub report: ExportReport,
}
impl core::fmt::Display for ExportError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(f, "export stopped: {}", self.error)
}
}
impl std::error::Error for ExportError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}
pub trait ExportSink {
fn begin(&mut self, header: &ExportHeader) -> Result<(), SinkError>;
fn region(&mut self, info: &RegionInfo) -> Result<(), SinkError>;
fn data(&mut self, chunk: Chunk) -> Result<(), SinkError>;
fn settle(&mut self) -> Result<(), SinkError> {
Ok(())
}
fn end_region(&mut self) -> Result<(), SinkError>;
fn finish(&mut self) -> Result<(), SinkError>;
}
struct Stop {
error: Error,
sink: Option<SinkError>,
}
impl From<Error> for Stop {
fn from(error: Error) -> Self {
Self { error, sink: None }
}
}
fn refused(cause: NotWrittenCause, region: Option<RegionId>) -> Error {
Error::NotWritten {
cause,
ctx: ErrorContext {
region,
..ErrorContext::default()
},
}
}
fn sunk(region: Option<RegionId>) -> impl Fn(SinkError) -> Stop {
move |e| {
let ctx = ErrorContext {
region,
..ErrorContext::default()
};
let error = match e {
SinkError::Io(raw) => Error::Io {
op: Op::Write,
raw,
ctx,
},
SinkError::Refused | SinkError::Other(_) => Error::NotWritten {
cause: NotWrittenCause::Interrupted,
ctx,
},
};
Stop {
error,
sink: Some(e),
}
}
}
fn too_many(e: &Error) -> bool {
matches!(
e,
Error::NotWritten {
cause: NotWrittenCause::TooManyInFlight,
..
}
)
}
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub fn export(&self, sink: &mut dyn ExportSink) -> Result<ExportReport, Box<ExportError>> {
self.export_with(sink, ExportOptions::default())
}
pub fn export_with(
&self,
sink: &mut dyn ExportSink,
opts: ExportOptions,
) -> Result<ExportReport, Box<ExportError>> {
let mut report = ExportReport {
writable: !self.inner.read_only,
..ExportReport::default()
};
match self.export_run(sink, &opts, &mut report) {
Ok(()) => {
report.region = None;
report.offset = 0;
report.complete = true;
Ok(report)
}
Err(Stop { error, sink }) => Err(Box::new(ExportError {
error,
sink,
report,
})),
}
}
fn export_plan(&self, opts: &ExportOptions) -> Result<CopyPlan, Error> {
if let Some(names) = &opts.regions {
if (1..names.len()).any(|i| names[..i].contains(&names[i])) {
return Err(not_written(
NotWrittenCause::InvalidOptions,
ErrorContext::default(),
));
}
}
let plan = self.plan_of(opts.accept_stale_pages, opts.regions.as_deref())?;
if !self.inner.read_only {
if let Some(f) = plan.facts.iter().find(|f| f.full.is_some() || f.superseded) {
return Err(refused(NotWrittenCause::InvalidOptions, Some(f.source.id)));
}
}
Ok(plan)
}
fn export_total(facts: &[Facts], block: u64) -> u64 {
let defaults = StoreOptions::default();
let log2_block = block.trailing_zeros() as u8;
let log2_table = defaults.log2_table_slot.clamp(log2_block, 16);
let count = u32::try_from(facts.len()).unwrap_or(u32::MAX);
let table =
crate::v2::layout_for(log2_block, log2_table, defaults.region_capacity.max(count))
.data_start();
facts.iter().fold(table, |total, f| {
let data = match f.kind {
RegionKind::Slot => max_object(log2_block) as u64,
_ => f.cap.unwrap_or(0).max(f.data_size),
};
total.saturating_add(2 * block).saturating_add(data)
})
}
fn export_run(
&self,
sink: &mut dyn ExportSink,
opts: &ExportOptions,
report: &mut ExportReport,
) -> Result<(), Stop> {
let plan = self.export_plan(opts)?;
let pace = self.inner.domain.new_pace();
let call = CallState::new(Arc::clone(&self.inner.qos), pace.clone());
let _bound = call::bind(&call);
let ran = self.export_body(sink, opts, report, &plan, pace, &call);
let failure = call.failure();
let clamp = |report: &mut ExportReport| {
if let Some(f) = &failure {
report.regions = report.regions.min(f.region as usize);
let base = call.base(f.region);
match plan.facts.get(f.region as usize).filter(|_| f.inside) {
Some(r) => {
let same = report.region.as_deref() == Some(r.name.as_str());
report.region = Some(r.name.clone());
report.offset = if same {
report.offset.min(f.offset)
} else {
f.offset
};
}
None => {
report.region = None;
report.offset = 0;
if let Some(b) = base {
report.bytes = report.bytes.min(b);
}
}
}
}
let inside = match &failure {
Some(f) => f.inside.then_some(f.region),
None => call.inside_region(),
};
if let (Some(i), Some(_)) = (inside, report.region.as_ref()) {
if let Some(w) = call.written_up_to(i) {
report.offset = report.offset.min(w);
}
if let Some(b) = call.base(i) {
report.bytes = report.bytes.min(b.saturating_add(report.offset));
}
}
if let Some(n) = call.regions_ended() {
let i = n as usize;
let named = report
.region
.as_ref()
.and_then(|name| plan.facts.iter().position(|f| &f.name == name));
if i < named.unwrap_or(report.regions) {
report.regions = i;
report.region = plan.facts.get(i).map(|f| f.name.clone());
report.offset = call.written_up_to(n).unwrap_or(0);
if let Some(b) = call.base(n) {
report.bytes = report.bytes.min(b.saturating_add(report.offset));
}
}
}
};
match ran {
Err(mut stop) => {
if let Some(f) = &failure {
stop.error = f.error.clone();
}
clamp(report);
Err(stop)
}
Ok(()) => {
if let Some(f) = &failure {
clamp(report);
return Err(Stop {
error: f.error.clone(),
sink: None,
});
}
if !call.all_finished() {
clamp(report);
return Err(Stop::from(not_written(
NotWrittenCause::Interrupted,
ErrorContext::default(),
)));
}
Ok(())
}
}
}
fn export_body(
&self,
sink: &mut dyn ExportSink,
opts: &ExportOptions,
report: &mut ExportReport,
plan: &CopyPlan,
pace: Option<crate::pace::Pace>,
call: &CallState,
) -> Result<(), Stop> {
report.facts = VolumeFacts {
omitted: plan.omitted.clone(),
lost_pages: plan.lost_pages.clone(),
stale_pages: plan.stale_pages.clone(),
verified_up_to: plan.verified_up_to.clone(),
superseded: plan.superseded.clone(),
};
let stream = self.export_stream(opts.depth, pace)?;
let header = ExportHeader {
region_count: u32::try_from(plan.facts.len()).unwrap_or(u32::MAX),
block_size: plan.block,
facts: report.facts.clone(),
writable: report.writable,
lend: stream.lend(),
total: Self::export_total(&plan.facts, plan.block),
};
sink.begin(&header).map_err(sunk(None))?;
if call.shares_source() {
stream.make_second();
}
for (i, f) in plan.facts.iter().enumerate() {
report.region = Some(f.name.clone());
report.offset = 0;
let index = u32::try_from(i).unwrap_or(u32::MAX);
call.enter(index, report.bytes);
self.export_region(f, plan.block, &stream, sink, report)
.map_err(|stop| Self::part_error(&stream, stop))?;
report.regions += 1;
call.leave(index.saturating_add(1));
}
report.region = None;
report.offset = 0;
if let Some(e) = stream.failure() {
return Err(Stop::from(e));
}
stream.cancel()?;
sink.finish().map_err(sunk(None))
}
fn part_error(stream: &ReadStream<P>, stop: Stop) -> Stop {
match stream.failure() {
Some(error) => Stop {
error,
sink: stop.sink,
},
None => stop,
}
}
fn object_at_turn(&self, r: &LiveRegion, block: u64) -> Result<Option<Vec<u8>>, Error> {
let header = r
.header
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let object = committed_object(&header, &self.inner.layout);
if object
.as_ref()
.is_some_and(|o| o.len() > max_object(block.trailing_zeros() as u8))
{
return Err(refused(NotWrittenCause::TooLarge, Some(r.id)));
}
Ok(object)
}
fn export_region(
&self,
f: &Facts,
block: u64,
stream: &ReadStream<P>,
sink: &mut dyn ExportSink,
report: &mut ExportReport,
) -> Result<(), Stop> {
let r = &f.source;
r.gate.live().map_err(|why| retired_error(why, r))?;
let id = Some(r.id);
let (durable_len, object, part) = match (f.kind, &f.full) {
(RegionKind::Slot, _) => (None, self.object_at_turn(r, block)?, None),
(_, Some((ranges, _))) => (
None,
None,
Some((f.data_size, Locator::Ranges(ranges.clone()))),
),
(kind, None) => {
let durable = (kind == RegionKind::Append && r.positioned.load(Ordering::Acquire))
.then(|| {
r.frontier
.as_ref()
.map_or(0, crate::frontier::AppendFrontier::durable)
.min(f.data_size)
});
(
durable,
None,
Some((durable.unwrap_or(f.data_size), Locator::Map)),
)
}
};
let info = RegionInfo {
name: f.name.clone(),
kind: f.kind,
generation: f.attrs.generation,
tag: f.attrs.tag,
cap: f.cap,
fill: f.attrs.fill,
fill_key: f.attrs.fill_key,
block_size: block,
data_size: f.data_size,
durable_len,
object_len: object.as_ref().map(|o| o.len() as u64),
full_copy: f.full.is_some(),
superseded: f.superseded,
};
sink.region(&info).map_err(sunk(id))?;
if let Some(o) = object.filter(|o| !o.is_empty()) {
let mut settled = false;
let chunk = loop {
match stream.lend_bytes(r, 0, &o) {
Ok(c) => break c,
Err(e) if too_many(&e) && !settled => {
sink.settle().map_err(sunk(id))?;
settled = true;
}
Err(e) => return Err(e.into()),
}
};
sink.data(chunk).map_err(sunk(id))?;
report.offset = o.len() as u64;
report.bytes += o.len() as u64;
}
if let Some((len, loc)) = part {
self.export_bytes(r, len, loc, stream, sink, report)?;
}
sink.end_region().map_err(sunk(id))
}
fn export_bytes(
&self,
r: &Arc<LiveRegion>,
len: u64,
loc: Locator,
stream: &ReadStream<P>,
sink: &mut dyn ExportSink,
report: &mut ExportReport,
) -> Result<(), Stop> {
let id = Some(r.id);
stream.add_part(r, 0..len, loc)?;
let mut got = 0u64;
let mut settled = false;
while got < len {
match stream.pull() {
Some(Ok(c)) => {
let n = c.bytes().len() as u64;
if n == 0 || c.offset() != got || n > len - got {
let mut e = crate::store::corrupt();
if let Error::Corruption { ctx, .. } = &mut e {
*ctx = ctx_range(r, got, len);
}
return Err(e.into());
}
settled = false;
sink.data(c).map_err(sunk(id))?;
got += n;
report.offset = got;
report.bytes += n;
}
Some(Err(e)) if too_many(&e) && !settled => {
sink.settle().map_err(sunk(id))?;
settled = true;
}
Some(Err(e)) => return Err(e.into()),
None => {
return Err(
not_written(NotWrittenCause::Interrupted, ctx_range(r, got, len)).into(),
);
}
}
}
Ok(())
}
}