use std::ops::ControlFlow;
use std::path::Path;
use store_io_core::errno::{is_already_exists, is_lock_held, is_not_found};
use store_io_core::error::{Error, ErrorContext, Named, NotWrittenCause, Op};
use store_io_core::id::RegionId;
use store_io_format::judge::Standing;
use store_io_format::meta::{FillPattern, REGION_META_LEN, RegionKind, RegionMeta};
use store_io_format::meta2::{FLAG_CAPPED, decode_region_header, max_object};
use store_io_format::slot::FORMAT_2;
use store_io_platform::{FileMode, FileName, IoOp, Platform};
use crate::exec::{Ran, clear_stale, run};
use crate::settle::settle_read;
use crate::store::{CONTAINER, LiveRegion, Store, StoreOptions, io, lock_meta};
use crate::{RegionOptions, ScanItem};
pub const COPYING: &str = "store.sio.copying";
pub const COPY_LOCK: &str = "store.sio.copy-lock";
#[derive(Debug, Clone, Default)]
pub struct CopyOptions {
pub store: StoreOptions,
pub accept_stale_pages: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CopyReport {
pub regions: usize,
pub bytes: u64,
pub source_format: u16,
pub omitted: Vec<RegionId>,
pub lost_pages: Vec<u32>,
pub stale_pages: Vec<u32>,
pub verified_up_to: Vec<(String, u64)>,
pub superseded: Vec<String>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct CopyAttrs {
pub(crate) generation: u64,
pub(crate) fill: FillPattern,
pub(crate) fill_key: [u8; 16],
pub(crate) tag: u32,
}
struct Facts {
name: String,
kind: RegionKind,
data_size: u64,
cap: Option<u64>,
attrs: CopyAttrs,
object: Option<Vec<u8>>,
full: Option<(Vec<(u64, u64)>, u64)>,
superseded: bool,
source: std::sync::Arc<LiveRegion>,
}
const FEED_PIECE: usize = 1 << 20;
const FEED_DEPTH: usize = 8;
struct Feed {
pieces: std::sync::mpsc::Receiver<Result<(u64, Vec<u8>), Error>>,
spent: std::sync::mpsc::Sender<Vec<u8>>,
current: Vec<u8>,
start: u64,
used: usize,
region: RegionId,
}
impl Feed {
fn start<'s, P: Platform>(
s: &'s std::thread::Scope<'s, '_>,
src: &'s Store<P>,
r: &'s LiveRegion,
len: u64,
) -> Result<(std::thread::ScopedJoinHandle<'s, Result<(), Error>>, Self), Error>
where
P::Queue: Send,
{
let (put, pieces) = std::sync::mpsc::sync_channel(FEED_DEPTH);
let (spent, back) = std::sync::mpsc::channel::<Vec<u8>>();
let handle = std::thread::Builder::new()
.name("store-io-copy".into())
.spawn_scoped(s, move || {
let mut fail = None;
let scanned = src.scan_region(r, 0, len, false, |item| match item {
ScanItem::Data { offset, bytes } => {
for (i, part) in bytes.chunks(FEED_PIECE).enumerate() {
let at = offset + (i * FEED_PIECE) as u64;
let mut v = back.try_recv().unwrap_or_default();
v.clear();
if v.try_reserve_exact(part.len()).is_err() {
fail = Some(crate::io::not_written(
NotWrittenCause::PoolExhausted,
crate::io::ctx_range(r, at, at + part.len() as u64),
));
return ControlFlow::Break(());
}
v.extend_from_slice(part);
if put.send(Ok((at, v))).is_err() {
return ControlFlow::Break(());
}
}
ControlFlow::Continue(())
}
ScanItem::Unreadable { start, end } => {
fail = Some(Error::Corruption {
kind: store_io_core::error::CorruptionKind::MediaError,
ctx: crate::io::ctx_range(r, start, end),
});
ControlFlow::Break(())
}
});
let failed = match (scanned, fail) {
(_, Some(e)) | (Err(e), None) => Some(e),
(Ok(_), None) => None,
};
if let Some(e) = failed {
let _sent = put.send(Err(e.clone()));
return Err(e);
}
Ok(())
})
.map_err(|e| Error::Io {
op: Op::Queue,
raw: os_error(&e),
ctx: ErrorContext {
region: Some(r.id),
..ErrorContext::default()
},
})?;
Ok((
handle,
Self {
pieces,
spent,
current: Vec::new(),
start: 0,
used: 0,
region: r.id,
},
))
}
fn read(&mut self, index: u64, dst: &mut [u8]) -> Result<(), Error> {
let mut done = 0usize;
while done < dst.len() {
if self.used == self.current.len() {
let (start, piece) = match self.pieces.recv() {
Ok(Ok(p)) => p,
Ok(Err(e)) => return Err(e),
Err(_) => return Err(self.wrong(index + done as u64, index + dst.len() as u64)),
};
let old = std::mem::replace(&mut self.current, piece);
let _recycled = self.spent.send(old);
self.start = start;
self.used = 0;
}
if self.start + self.used as u64 != index + done as u64 {
return Err(self.wrong(index + done as u64, index + dst.len() as u64));
}
let n = (self.current.len() - self.used).min(dst.len() - done);
dst[done..done + n].copy_from_slice(&self.current[self.used..self.used + n]);
self.used += n;
done += n;
}
Ok(())
}
fn wrong(&self, start: u64, end: u64) -> Error {
let mut e = crate::store::corrupt();
if let Error::Corruption { ctx, .. } = &mut e {
*ctx = ErrorContext {
region: Some(self.region),
range: Some(store_io_core::error::ByteRange { start, end }),
};
}
e
}
}
fn os_error(e: &std::io::Error) -> store_io_core::error::OsError {
use store_io_core::error::{OsError, OsErrorSource};
OsError {
code: e.raw_os_error().unwrap_or(0),
source: if cfg!(windows) {
OsErrorSource::Win32
} else {
OsErrorSource::Errno
},
}
}
pub fn copy_store<P: Platform + Clone>(
platform: P,
from: &Path,
to: &Path,
opts: CopyOptions,
) -> Result<CopyReport, Error>
where
P::Queue: Send,
{
let src = Store::open_readonly(platform.clone(), from, opts.store.clone())?;
let report = src.report();
let refused = |cause, region| Error::NotWritten {
cause,
ctx: ErrorContext {
region,
..ErrorContext::default()
},
};
let accept = opts.accept_stale_pages;
let (mut omitted, mut lost_pages, mut stale_pages) = (Vec::new(), Vec::new(), Vec::new());
if accept {
omitted = src.omitted_for_copy();
lost_pages.clone_from(&report.metadata.lost_pages);
stale_pages.clone_from(&report.metadata.stale_pages);
} else {
if let Some(u) = src.unavailable_regions().first() {
return Err(refused(NotWrittenCause::NotReady, Some(u.id)));
}
if let Some(&(p, s)) = report.metadata.invalid_entries.first() {
let id = RegionId::from_raw((u64::from(p) << 32) | u64::from(s));
return Err(refused(NotWrittenCause::NotReady, Some(id)));
}
if src.frozen() {
return Err(refused(NotWrittenCause::Frozen, None));
}
}
let block = report.block_size;
let log2 = block.trailing_zeros() as u8;
let source_format = report.format;
let regions = src.regions_snapshot();
let mut facts = Vec::with_capacity(regions.len());
let (mut verified_up_to, mut superseded) = (Vec::new(), Vec::new());
for r in ®ions {
let f = src.facts_for_copy(r, ®ions)?;
if let Some((_, recorded)) = &f.full {
verified_up_to.push((f.name.clone(), *recorded));
}
if f.superseded {
superseded.push(f.name.clone());
}
if f.object
.as_ref()
.is_some_and(|o| o.len() > max_object(log2))
{
return Err(refused(NotWrittenCause::TooLarge, Some(r.id)));
}
facts.push(f);
}
let dir = platform
.open_dir(to, true)
.map_err(|raw| io(Op::Create, raw))?;
let final_name = name(CONTAINER)?;
let temp = name(COPYING)?;
let holds_store = || match platform.open_file(&dir, &final_name, FileMode::ReadOnly) {
Ok(_) => Err(Error::AlreadyExists { what: Named::Store }),
Err(raw) if is_not_found(raw) => Ok(()),
Err(raw) => Err(io(Op::Open, raw)),
};
holds_store()?;
let lock_name = name(COPY_LOCK)?;
let lock_file = match platform.create_file(&dir, &lock_name) {
Ok(f) => f,
Err(raw) if is_already_exists(raw) => platform
.open_file(&dir, &lock_name, FileMode::ReadWrite)
.map_err(|raw| io(Op::Open, raw))?,
Err(raw) => return Err(io(Op::Create, raw)),
};
let _lock = platform.lock_exclusive(&lock_file).map_err(|raw| {
if is_lock_held(raw) {
Error::Locked
} else {
io(Op::Lock, raw)
}
})?;
holds_store()?;
match platform.unlink(&dir, &temp) {
Ok(()) => {}
Err(raw) if is_not_found(raw) => {}
Err(raw) => return Err(io(Op::Unlink, raw)),
}
let mut target_opts = opts.store.clone();
target_opts.container_format = FORMAT_2;
target_opts.block_size = Some(block as u32);
target_opts.region_capacity = target_opts
.region_capacity
.max(u32::try_from(facts.len()).unwrap_or(u32::MAX));
let mut bytes = 0u64;
{
let dst = Store::create_named(platform.clone(), to, target_opts, &temp)?;
if dst.report().block_size != block {
return Err(refused(NotWrittenCause::Misaligned, None));
}
for f in &facts {
bytes += dst.recreate(&src, f)?;
}
dst.flush_all_accounted()?;
}
let file = platform
.open_file(&dir, &temp, FileMode::ReadWrite)
.map_err(|raw| io(Op::Open, raw))?;
match platform.rename_noreplace(&dir, &temp, &final_name, &file) {
Ok(()) => {}
Err(raw) if is_already_exists(raw) => {
drop(file);
if platform.unlink(&dir, &temp).is_ok() {
let _synced = platform.sync_dir(&dir);
}
return Err(Error::AlreadyExists { what: Named::Store });
}
Err(raw) => {
return Err(Error::DurabilityUnknown {
op: Op::Rename,
raw: Some(raw),
ctx: ErrorContext::default(),
});
}
}
drop(file);
platform
.sync_dir(&dir)
.map_err(|raw| Error::DurabilityUnknown {
op: Op::SyncDir,
raw: Some(raw),
ctx: ErrorContext::default(),
})?;
Ok(CopyReport {
regions: facts.len(),
bytes,
source_format,
omitted,
lost_pages,
stale_pages,
verified_up_to,
superseded,
})
}
fn name(s: &str) -> Result<FileName, Error> {
FileName::new(s).map_err(|_| Error::NotWritten {
cause: NotWrittenCause::InvalidName,
ctx: ErrorContext::default(),
})
}
const CHUNK: usize = 1 << 20;
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
fn omitted_for_copy(&self) -> Vec<RegionId> {
let meta = lock_meta(&self.inner.meta);
match meta.v2.as_deref() {
Some(v2) => {
let mut ids: Vec<RegionId> = v2
.holds
.iter()
.map(|h| RegionId::from_raw(h.id()))
.collect();
ids.sort_unstable_by_key(|id| id.get());
ids
}
None => {
let mut ids: Vec<RegionId> = meta.unavailable.iter().map(|u| u.id).collect();
ids.sort_unstable_by_key(|id| id.get());
ids
}
}
}
fn full_copy(
&self,
r: &LiveRegion,
regions: &[std::sync::Arc<LiveRegion>],
) -> Option<(Vec<(u64, u64)>, u64)> {
let meta = lock_meta(&self.inner.meta);
let v2 = meta.v2.as_deref()?;
let page = usize::try_from(r.id.get() >> 32).ok()?;
if v2.pages.get(page)?.standing != Standing::Unconfirmed {
return None;
}
let block = self.block();
let mut ranges: Vec<(u64, u64)> = r.map.extents().collect();
let overlaps = |(a, n): (u64, u64), (b, m): (u64, u64)| a < b + m && b < a + n;
let extra = v2
.stale_extra
.iter()
.find(|e| e.1 == r.id.get())
.map(|e| e.2);
let lost = v2
.pages
.iter()
.any(|p| p.standing != Standing::Confirmed && p.standing != Standing::Unconfirmed);
if let Some(extra) = extra.filter(|_| !lost) {
let mut others = regions
.iter()
.flat_map(|o| {
o.map
.extents()
.chain(o.header_at.iter().map(|&h| (h, block)))
.collect::<Vec<_>>()
})
.chain(core::iter::once((0, self.inner.layout.data_start())))
.chain(v2.holds.iter().filter_map(|h| h.span))
.chain(v2.holds.iter().flat_map(|h| h.own.iter().copied()))
.chain(
v2.stale_extra
.iter()
.filter(|e| e.1 != r.id.get())
.map(|e| e.2),
);
if !others.any(|o| overlaps(extra, o)) {
ranges.push(extra);
}
}
Some((ranges, r.map.len()))
}
fn read_raw(
&self,
r: &LiveRegion,
mut at: u64,
mut offset: u64,
out: &mut [u8],
) -> Result<(), Error> {
let mut lane = self.inner.queues.get(0);
clear_stale(&mut lane).map_err(|raw| io(Op::Read, raw))?;
let chunk = self.chunk();
for piece in out.chunks_mut(chunk) {
let c = crate::io::ctx_range(r, at, at + piece.len() as u64);
let buf = self.take(piece.len(), c)?;
let op = IoOp::Read {
file: &self.inner.file,
offset,
buf,
};
let (got, result) = match run(&mut lane, op) {
Ok(Ran::Done { buf, result }) => (buf, result),
Ok(Ran::Abandoned(raw)) | Err((_, Some(raw))) => return Err(io(Op::Read, raw)),
Err((_, None)) => {
return Err(crate::io::not_written(NotWrittenCause::QueueFull, c));
}
};
let n = settle_read(self, r, at, at + piece.len() as u64, got.as_ref(), result)?;
match got {
Some(b) if n == piece.len() => piece.copy_from_slice(&b.as_slice()[..n]),
_ => return Err(crate::store::corrupt()),
}
offset += piece.len() as u64;
at += piece.len() as u64;
}
Ok(())
}
fn at_risk(&self, r: &LiveRegion) -> bool {
let meta = lock_meta(&self.inner.meta);
let Some(v2) = meta.v2.as_deref() else {
return false;
};
let page = usize::try_from(r.id.get() >> 32).unwrap_or(usize::MAX);
v2.pages.get(page).is_some_and(|pg| {
(pg.standing == Standing::Unconfirmed || pg.pair.possibly_accepted())
&& v2.pages.iter().enumerate().any(|(i, o)| {
i != page && (o.standing != Standing::Confirmed || o.pair.possibly_accepted())
})
})
}
fn facts_for_copy(
&self,
r: &std::sync::Arc<LiveRegion>,
regions: &[std::sync::Arc<LiveRegion>],
) -> Result<Facts, Error> {
let inner = &self.inner;
let header = r
.header
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let payload = header.payload.as_deref().unwrap_or(&[]);
let format_2 = inner.layout.format == FORMAT_2;
let (fill, fill_key, capped) = if format_2 {
let v = decode_region_header(payload, inner.layout.log2_block, u64::MAX)
.map_err(|_| crate::store::corrupt())?;
(
v.header.fill,
v.header.fill_key,
v.header.flags & FLAG_CAPPED != 0,
)
} else {
let m = payload
.get(..REGION_META_LEN)
.and_then(|meta| RegionMeta::decode(meta).ok())
.ok_or_else(crate::store::corrupt)?;
(m.fill, m.fill_key, false)
};
let generation = if !format_2 && r.kind == RegionKind::Slot {
1
} else {
r.generation.get()
};
let tag = {
let meta = lock_meta(&inner.meta);
match meta.v2.as_deref() {
Some(v2) => {
let id = r.id.get();
v2.pages
.get(usize::try_from(id >> 32).unwrap_or(usize::MAX))
.and_then(|p| {
p.entries
.iter()
.find(|e| e.seq == (id & 0xFFFF_FFFF) as u32)
})
.map(|e| e.tag)
.ok_or_else(crate::store::corrupt)?
}
None => meta
.entries
.iter()
.find(|e| u64::from(e.region_id) == r.id.get())
.map(|e| e.tag)
.ok_or_else(crate::store::corrupt)?,
}
};
let object = if r.kind == RegionKind::Slot {
self.slot(r.name.as_str())?.read()
} else {
None
};
let full = if r.kind == RegionKind::Slot {
None
} else {
self.full_copy(r, regions)
};
let data_size = full
.as_ref()
.map_or(r.map.len(), |(ranges, _)| ranges.iter().map(|x| x.1).sum());
Ok(Facts {
name: r.name.as_str().to_owned(),
kind: r.kind,
data_size,
cap: capped.then(|| r.map.capacity().max(data_size)),
attrs: CopyAttrs {
generation,
fill,
fill_key,
tag,
},
object,
full,
superseded: r.superseded || self.at_risk(r),
source: std::sync::Arc::clone(r),
})
}
fn recreate(&self, src: &Store<P>, f: &Facts) -> Result<u64, Error> {
let mut opts = RegionOptions::default().size(f.data_size);
if let Some(cap) = f.cap {
opts = opts.max_size(cap);
}
if f.kind != RegionKind::Slot && f.data_size % self.block() != 0 {
return Err(Error::NotWritten {
cause: NotWrittenCause::Misaligned,
ctx: ErrorContext {
region: Some(f.source.id),
..ErrorContext::default()
},
});
}
if f.full.is_none() && f.kind != RegionKind::Slot && f.data_size > 0 {
let made = std::thread::scope(|s| {
let mut feed = None;
let mut content = |index: u64, dst: &mut [u8]| -> Result<(), Error> {
if dst.is_empty() {
return Ok(());
}
if feed.is_none() {
feed = Some(Feed::start(s, src, &f.source, f.data_size)?);
}
match feed.as_mut() {
Some((_, reader)) => reader.read(index, dst),
None => Err(crate::store::corrupt()),
}
};
let made = self.provision2(
&f.name,
f.kind,
opts,
None,
Some(&f.attrs),
Some(&mut content),
);
let fed = match feed.take() {
None => Ok(()),
Some((handle, reader)) => {
drop(reader);
match handle.join() {
Ok(fed) => fed,
Err(panic) => std::panic::resume_unwind(panic),
}
}
};
match (made, fed) {
(Ok(made), Ok(())) => Ok(made),
(Err(e), _) | (Ok(_), Err(e)) => Err(e),
}
})?;
debug_assert_eq!(made.generation.get(), f.attrs.generation, "{}", f.name);
if made.map.len() != f.data_size {
return Err(crate::store::corrupt());
}
return Ok(f.data_size);
}
let made = self.provision2(&f.name, f.kind, opts, None, Some(&f.attrs), None)?;
debug_assert_eq!(made.generation.get(), f.attrs.generation, "{}", f.name);
debug_assert_eq!(made.map.len(), f.data_size, "{}", f.name);
if let Some((ranges, _)) = &f.full {
let mut buf = vec![0u8; CHUNK];
let mut at = 0u64;
for &(file, len) in ranges {
let mut done = 0u64;
while done < len {
let n = (len - done).min(CHUNK as u64) as usize;
src.read_raw(&f.source, at, file + done, &mut buf[..n])?;
match f.kind {
RegionKind::Page => {
let to = self.page_region(&f.name)?;
let _ticket = to.write(to.pos(at)?, &buf[..n])?;
}
_ => {
let _ticket = self.append_region(&f.name)?.append(&buf[..n])?;
}
}
done += n as u64;
at += n as u64;
}
}
return Ok(f.data_size);
}
match f.kind {
RegionKind::Slot => {
let Some(object) = &f.object else {
return Ok(0);
};
let _receipt = self.slot(&f.name)?.commit(object)?;
Ok(object.len() as u64)
}
RegionKind::Page | RegionKind::Append => {
if f.data_size != 0 {
return Err(crate::store::corrupt());
}
Ok(0)
}
}
}
}