use std::ops::ControlFlow;
use std::path::Path;
use store_io_core::error::{Error, ErrorContext, NotWrittenCause, Op};
use store_io_core::id::RegionId;
use store_io_format::judge::Standing;
use store_io_format::meta::{FillPattern, REGION_META_LEN, RegionKind, RegionMeta};
use store_io_format::meta2::{FLAG_CAPPED, decode_region_header, max_object};
use store_io_format::slot::FORMAT_2;
use store_io_platform::{IoOp, Platform};
use crate::exec::{Ran, clear_stale, run};
use crate::pace::Pace;
use crate::qos::Qos;
use crate::settle::settle_read;
use crate::target::TargetDir;
use crate::slot_handle::committed_object;
use crate::store::{LiveRegion, Meta, Store, StoreOptions, io, lock_meta};
use crate::{RegionOptions, ScanItem};
pub const COPYING: &str = "store.sio.copying";
pub const COPY_LOCK: &str = "store.sio.copy-lock";
#[derive(Debug, Clone, Default)]
pub struct CopyOptions {
pub store: StoreOptions,
pub accept_stale_pages: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CopyReport {
pub regions: usize,
pub bytes: u64,
pub source_format: u16,
pub omitted: Vec<RegionId>,
pub lost_pages: Vec<u32>,
pub stale_pages: Vec<u32>,
pub verified_up_to: Vec<(String, u64)>,
pub superseded: Vec<String>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct CopyAttrs {
pub(crate) generation: u64,
pub(crate) fill: FillPattern,
pub(crate) fill_key: [u8; 16],
pub(crate) tag: u32,
}
struct HeaderFacts {
fill: FillPattern,
fill_key: [u8; 16],
capped: bool,
object: Option<Vec<u8>>,
}
pub(crate) struct CopyPlan {
pub(crate) facts: Vec<Facts>,
pub(crate) omitted: Vec<RegionId>,
pub(crate) lost_pages: Vec<u32>,
pub(crate) stale_pages: Vec<u32>,
pub(crate) verified_up_to: Vec<(String, u64)>,
pub(crate) superseded: Vec<String>,
pub(crate) block: u64,
pub(crate) source_format: u16,
}
fn refused(cause: NotWrittenCause, region: Option<RegionId>) -> Error {
Error::NotWritten {
cause,
ctx: ErrorContext {
region,
..ErrorContext::default()
},
}
}
pub(crate) struct Facts {
pub(crate) name: String,
pub(crate) kind: RegionKind,
pub(crate) data_size: u64,
pub(crate) cap: Option<u64>,
pub(crate) attrs: CopyAttrs,
pub(crate) object: Option<Vec<u8>>,
pub(crate) full: Option<(Vec<(u64, u64)>, u64)>,
pub(crate) superseded: bool,
pub(crate) source: std::sync::Arc<LiveRegion>,
}
const FEED_PIECE: usize = 1 << 20;
const FEED_DEPTH: usize = 8;
struct Feed {
pieces: std::sync::mpsc::Receiver<Result<(u64, Vec<u8>), Error>>,
spent: std::sync::mpsc::Sender<Vec<u8>>,
current: Vec<u8>,
start: u64,
used: usize,
region: RegionId,
}
impl Feed {
fn start<'s, P: Platform>(
s: &'s std::thread::Scope<'s, '_>,
src: &'s Store<P>,
r: &'s LiveRegion,
len: u64,
pace: Option<Pace>,
) -> Result<(std::thread::ScopedJoinHandle<'s, Result<(), Error>>, Self), Error>
where
P::Queue: Send,
{
let (put, pieces) = std::sync::mpsc::sync_channel(FEED_DEPTH);
let (spent, back) = std::sync::mpsc::channel::<Vec<u8>>();
let handle = std::thread::Builder::new()
.name("store-io-copy".into())
.spawn_scoped(s, move || {
let mut fail = None;
let scanned =
src.scan_region(r, 0, len, false, pace.as_ref(), |item, drain| match item {
ScanItem::Data { offset, bytes } => {
for (i, part) in bytes.chunks(FEED_PIECE).enumerate() {
let at = offset + (i * FEED_PIECE) as u64;
let mut v = back.try_recv().unwrap_or_default();
v.clear();
if v.try_reserve_exact(part.len()).is_err() {
fail = Some(crate::io::not_written(
NotWrittenCause::PoolExhausted,
crate::io::ctx_range(r, at, at + part.len() as u64),
));
return ControlFlow::Break(());
}
v.extend_from_slice(part);
let sent = match put.try_send(Ok((at, v))) {
Ok(()) => true,
Err(std::sync::mpsc::TrySendError::Full(piece)) => {
drain() && {
crate::test_hook::step(
crate::test_hook::Step::FeedBlocking,
);
put.send(piece).is_ok()
}
}
Err(std::sync::mpsc::TrySendError::Disconnected(_)) => false,
};
if !sent {
return ControlFlow::Break(());
}
}
ControlFlow::Continue(())
}
ScanItem::Unreadable { start, end } => {
fail = Some(Error::Corruption {
kind: store_io_core::error::CorruptionKind::MediaError,
ctx: crate::io::ctx_range(r, start, end),
});
ControlFlow::Break(())
}
});
let failed = match (scanned, fail) {
(_, Some(e)) | (Err(e), None) => Some(e),
(Ok(_), None) => None,
};
if let Some(e) = failed {
let _sent = put.send(Err(e.clone()));
return Err(e);
}
Ok(())
})
.map_err(|e| Error::Io {
op: Op::Queue,
raw: os_error(&e),
ctx: ErrorContext {
region: Some(r.id),
..ErrorContext::default()
},
})?;
Ok((
handle,
Self {
pieces,
spent,
current: Vec::new(),
start: 0,
used: 0,
region: r.id,
},
))
}
fn read(&mut self, index: u64, dst: &mut [u8]) -> Result<(), Error> {
let mut done = 0usize;
while done < dst.len() {
if self.used == self.current.len() {
let (start, piece) = match self.pieces.recv() {
Ok(Ok(p)) => p,
Ok(Err(e)) => return Err(e),
Err(_) => return Err(self.wrong(index + done as u64, index + dst.len() as u64)),
};
let old = std::mem::replace(&mut self.current, piece);
let _recycled = self.spent.send(old);
self.start = start;
self.used = 0;
}
if self.start + self.used as u64 != index + done as u64 {
return Err(self.wrong(index + done as u64, index + dst.len() as u64));
}
let n = (self.current.len() - self.used).min(dst.len() - done);
dst[done..done + n].copy_from_slice(&self.current[self.used..self.used + n]);
self.used += n;
done += n;
}
Ok(())
}
fn wrong(&self, start: u64, end: u64) -> Error {
let mut e = crate::store::corrupt();
if let Error::Corruption { ctx, .. } = &mut e {
*ctx = ErrorContext {
region: Some(self.region),
range: Some(store_io_core::error::ByteRange { start, end }),
};
}
e
}
}
fn os_error(e: &std::io::Error) -> store_io_core::error::OsError {
use store_io_core::error::{OsError, OsErrorSource};
OsError {
code: e.raw_os_error().unwrap_or(0),
source: if cfg!(windows) {
OsErrorSource::Win32
} else {
OsErrorSource::Errno
},
}
}
pub fn copy_store<P: Platform + Clone>(
platform: P,
from: &Path,
to: &Path,
opts: CopyOptions,
) -> Result<CopyReport, Error>
where
P::Queue: Send,
{
let src = Store::open_readonly(platform.clone(), from, opts.store.clone())?;
let CopyPlan {
facts,
omitted,
lost_pages,
stale_pages,
verified_up_to,
superseded,
block,
source_format,
} = src.copy_plan(opts.accept_stale_pages)?;
let target = TargetDir::open(&platform, to)?;
let mut target_opts = opts.store.clone();
target_opts.container_format = FORMAT_2;
target_opts.block_size = Some(block as u32);
target_opts.region_capacity = target_opts
.region_capacity
.max(u32::try_from(facts.len()).unwrap_or(u32::MAX));
let mut bytes = 0u64;
{
let dst = Store::create_named(
platform.clone(),
to,
target_opts,
&target.temp,
store_io_core::IoClass::Background,
)?;
if dst.report().block_size != block {
return Err(refused(NotWrittenCause::Misaligned, None));
}
let reading = dst.reader_pace(&src);
for f in &facts {
bytes += dst.recreate(&src, f, reading.as_ref())?;
}
dst.flush_all_accounted()?;
}
target.commit(&platform)?;
Ok(CopyReport {
regions: facts.len(),
bytes,
source_format,
omitted,
lost_pages,
stale_pages,
verified_up_to,
superseded,
})
}
const CHUNK: usize = 1 << 20;
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
fn omitted_for_copy(meta: &Meta) -> Vec<RegionId> {
match meta.v2.as_deref() {
Some(v2) => {
let mut ids: Vec<RegionId> = v2
.holds
.iter()
.map(|h| RegionId::from_raw(h.id()))
.collect();
ids.sort_unstable_by_key(|id| id.get());
ids
}
None => {
let mut ids: Vec<RegionId> = meta.unavailable.iter().map(|u| u.id).collect();
ids.sort_unstable_by_key(|id| id.get());
ids
}
}
}
fn full_copy(
&self,
meta: &Meta,
r: &LiveRegion,
regions: &[std::sync::Arc<LiveRegion>],
) -> Option<(Vec<(u64, u64)>, u64)> {
let v2 = meta.v2.as_deref()?;
let page = usize::try_from(r.id.get() >> 32).ok()?;
if v2.pages.get(page)?.standing != Standing::Unconfirmed {
return None;
}
let block = self.block();
let mut ranges: Vec<(u64, u64)> = r.map.extents().collect();
let overlaps = |(a, n): (u64, u64), (b, m): (u64, u64)| a < b + m && b < a + n;
let extra = v2
.stale_extra
.iter()
.find(|e| e.1 == r.id.get())
.map(|e| e.2);
let lost = v2
.pages
.iter()
.any(|p| p.standing != Standing::Confirmed && p.standing != Standing::Unconfirmed);
if let Some(extra) = extra.filter(|_| !lost) {
let mut others = regions
.iter()
.flat_map(|o| {
o.map
.extents()
.chain(o.header_at.iter().map(|&h| (h, block)))
.collect::<Vec<_>>()
})
.chain(core::iter::once((0, self.inner.layout.data_start())))
.chain(v2.holds.iter().filter_map(|h| h.span))
.chain(v2.holds.iter().flat_map(|h| h.own.iter().copied()))
.chain(
v2.stale_extra
.iter()
.filter(|e| e.1 != r.id.get())
.map(|e| e.2),
);
if !others.any(|o| overlaps(extra, o)) {
ranges.push(extra);
}
}
Some((ranges, r.map.len()))
}
fn read_raw(
&self,
r: &LiveRegion,
mut at: u64,
mut offset: u64,
out: &mut [u8],
pace: Option<&Pace>,
) -> Result<(), Error> {
let mut lane = self.inner.queues.get(0);
clear_stale(&mut lane).map_err(|raw| io(Op::Read, raw))?;
let chunk = pace.map_or(self.chunk(), |p| p.piece().min(self.chunk()));
for piece in out.chunks_mut(chunk) {
let c = crate::io::ctx_range(r, at, at + piece.len() as u64);
let buf = match pace {
Some(_) => self.take_capped(piece.len(), c)?,
None => self.take(piece.len(), c)?,
};
let op = IoOp::Read {
file: &self.inner.file,
offset,
buf,
};
let admitted = pace.map(Pace::admit).transpose()?;
let ran = run(&mut lane, op);
drop(admitted);
let (got, result) = match ran {
Ok(Ran::Done { buf, result }) => (buf, result),
Ok(Ran::Abandoned(raw)) | Err((_, Some(raw))) => return Err(io(Op::Read, raw)),
Err((_, None)) => {
return Err(crate::io::not_written(NotWrittenCause::QueueFull, c));
}
};
let n = settle_read(self, r, at, at + piece.len() as u64, got.as_ref(), result)?;
match got {
Some(b) if n == piece.len() => piece.copy_from_slice(&b.as_slice()[..n]),
_ => return Err(crate::store::corrupt()),
}
offset += piece.len() as u64;
at += piece.len() as u64;
}
Ok(())
}
fn reader_pace(&self, src: &Store<P>) -> Option<Pace> {
let own = self.inner.domain.pace()?;
if Qos::same_group(&self.inner.qos, &src.inner.qos) {
Some(own.second())
} else {
src.inner.domain.new_pace()
}
}
fn at_risk(meta: &Meta, r: &LiveRegion) -> bool {
let Some(v2) = meta.v2.as_deref() else {
return false;
};
let page = usize::try_from(r.id.get() >> 32).unwrap_or(usize::MAX);
v2.pages.get(page).is_some_and(|pg| {
(pg.standing == Standing::Unconfirmed || pg.pair.possibly_accepted())
&& v2.pages.iter().enumerate().any(|(i, o)| {
i != page && (o.standing != Standing::Confirmed || o.pair.possibly_accepted())
})
})
}
fn header_facts(&self, r: &LiveRegion) -> Result<HeaderFacts, Error> {
let inner = &self.inner;
let header = r
.header
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
if r.gate.live().is_err() {
return Err(refused(NotWrittenCause::StaleGeneration, Some(r.id)));
}
let payload = header.payload.as_deref().unwrap_or(&[]);
let format_2 = inner.layout.format == FORMAT_2;
let (fill, fill_key, capped) = if format_2 {
let v = decode_region_header(payload, inner.layout.log2_block, u64::MAX)
.map_err(|_| crate::store::corrupt())?;
(
v.header.fill,
v.header.fill_key,
v.header.flags & FLAG_CAPPED != 0,
)
} else {
let m = payload
.get(..REGION_META_LEN)
.and_then(|meta| RegionMeta::decode(meta).ok())
.ok_or_else(crate::store::corrupt)?;
(m.fill, m.fill_key, false)
};
let object = if r.kind == RegionKind::Slot {
committed_object(&header, &inner.layout)
} else {
None
};
Ok(HeaderFacts {
fill,
fill_key,
capped,
object,
})
}
fn tag_of(meta: &Meta, r: &LiveRegion) -> Result<u32, Error> {
match meta.v2.as_deref() {
Some(v2) => {
let id = r.id.get();
v2.pages
.get(usize::try_from(id >> 32).unwrap_or(usize::MAX))
.and_then(|p| {
p.entries
.iter()
.find(|e| e.seq == (id & 0xFFFF_FFFF) as u32)
})
.map(|e| e.tag)
.ok_or_else(crate::store::corrupt)
}
None => meta
.entries
.iter()
.find(|e| u64::from(e.region_id) == r.id.get())
.map(|e| e.tag)
.ok_or_else(crate::store::corrupt),
}
}
fn facts_for_copy(
&self,
meta: &Meta,
r: &std::sync::Arc<LiveRegion>,
regions: &[std::sync::Arc<LiveRegion>],
h: HeaderFacts,
) -> Result<Facts, Error> {
let format_2 = self.inner.layout.format == FORMAT_2;
let generation = if !format_2 && r.kind == RegionKind::Slot {
1
} else {
r.generation.get()
};
let tag = Self::tag_of(meta, r)?;
let full = if r.kind == RegionKind::Slot {
None
} else {
self.full_copy(meta, r, regions)
};
let data_size = full
.as_ref()
.map_or(r.map.len(), |(ranges, _)| ranges.iter().map(|x| x.1).sum());
Ok(Facts {
name: r.name.as_str().to_owned(),
kind: r.kind,
data_size,
cap: h.capped.then(|| r.map.capacity().max(data_size)),
attrs: CopyAttrs {
generation,
fill: h.fill,
fill_key: h.fill_key,
tag,
},
object: h.object,
full,
superseded: r.superseded || Self::at_risk(meta, r),
source: std::sync::Arc::clone(r),
})
}
pub(crate) fn copy_plan(&self, accept: bool) -> Result<CopyPlan, Error> {
self.plan_of(accept, None)
}
pub(crate) fn plan_of(
&self,
accept: bool,
subset: Option<&[String]>,
) -> Result<CopyPlan, Error> {
let report = self.report();
let block = report.block_size;
let log2 = block.trailing_zeros() as u8;
let picked =
|r: &LiveRegion| subset.is_none_or(|names| names.iter().any(|n| n == r.name.as_str()));
let regions: Vec<_> = self
.regions_snapshot()
.into_iter()
.filter(|r| picked(r))
.collect();
let mut headers = Vec::with_capacity(regions.len());
for r in ®ions {
headers.push(self.header_facts(r)?);
}
crate::test_hook::step(crate::test_hook::Step::PlanHeaders);
let meta = lock_meta(&self.inner.meta);
let all = self.regions_snapshot();
let held: Vec<_> = all.iter().filter(|r| picked(r)).cloned().collect();
let same = |a: &LiveRegion, b: &LiveRegion| a.id == b.id && a.generation == b.generation;
if let Some(i) = (0..regions.len().max(held.len()))
.find(|&i| !matches!((regions.get(i), held.get(i)), (Some(a), Some(b)) if same(a, b)))
{
let changed = regions.get(i).or_else(|| held.get(i)).map(|r| r.id);
return Err(refused(NotWrittenCause::StaleGeneration, changed));
}
if let Some(names) = subset {
if names.is_empty() {
return Err(refused(NotWrittenCause::InvalidOptions, None));
}
if let Some(n) = names
.iter()
.find(|n| !held.iter().any(|r| r.name.as_str() == n.as_str()))
{
if let Some(u) = meta.unavailable.iter().find(|u| u.name == **n) {
return Err(refused(NotWrittenCause::NotReady, Some(u.id)));
}
return Err(refused(NotWrittenCause::InvalidOptions, None));
}
}
let (mut omitted, mut lost_pages, mut stale_pages) = (Vec::new(), Vec::new(), Vec::new());
if accept {
omitted = Self::omitted_for_copy(&meta);
if let Some(v2) = meta.v2.as_deref() {
for (i, p) in v2.pages.iter().enumerate() {
match p.standing {
Standing::Unconfirmed => stale_pages.push(i as u32),
Standing::Lost | Standing::Refused => lost_pages.push(i as u32),
Standing::Confirmed => {}
}
}
}
} else {
if let Some(u) = meta.unavailable.first() {
return Err(refused(NotWrittenCause::NotReady, Some(u.id)));
}
let still = |id: u64| {
meta.v2
.as_deref()
.is_none_or(|v2| v2.holds.iter().any(|h| h.id() == id))
};
if let Some(id) = report
.metadata
.invalid_entries
.iter()
.map(|&(p, s)| (u64::from(p) << 32) | u64::from(s))
.find(|&id| still(id))
{
return Err(refused(
NotWrittenCause::NotReady,
Some(RegionId::from_raw(id)),
));
}
if self.frozen() {
return Err(refused(NotWrittenCause::Frozen, None));
}
}
let mut facts = Vec::with_capacity(held.len());
let (mut verified_up_to, mut superseded) = (Vec::new(), Vec::new());
for (r, h) in held.iter().zip(headers) {
if h.object
.as_ref()
.is_some_and(|o| o.len() > max_object(log2))
{
return Err(refused(NotWrittenCause::TooLarge, Some(r.id)));
}
let f = self.facts_for_copy(&meta, r, &all, h)?;
if let Some((_, recorded)) = &f.full {
verified_up_to.push((f.name.clone(), *recorded));
}
if f.superseded {
superseded.push(f.name.clone());
}
facts.push(f);
}
drop(meta);
Ok(CopyPlan {
facts,
omitted,
lost_pages,
stale_pages,
verified_up_to,
superseded,
block,
source_format: report.format,
})
}
fn recreate(&self, src: &Store<P>, f: &Facts, reading: Option<&Pace>) -> Result<u64, Error> {
let mut opts = RegionOptions::default().size(f.data_size);
if let Some(cap) = f.cap {
opts = opts.max_size(cap);
}
if f.kind != RegionKind::Slot && f.data_size % self.block() != 0 {
return Err(Error::NotWritten {
cause: NotWrittenCause::Misaligned,
ctx: ErrorContext {
region: Some(f.source.id),
..ErrorContext::default()
},
});
}
if f.full.is_none() && f.kind != RegionKind::Slot && f.data_size > 0 {
let made = std::thread::scope(|s| {
let mut feed = None;
let mut content = |index: u64, dst: &mut [u8]| -> Result<(), Error> {
if dst.is_empty() {
return Ok(());
}
if feed.is_none() {
feed = Some(Feed::start(
s,
src,
&f.source,
f.data_size,
reading.cloned(),
)?);
}
match feed.as_mut() {
Some((_, reader)) => reader.read(index, dst),
None => Err(crate::store::corrupt()),
}
};
let made = self.provision2(
&f.name,
f.kind,
opts,
None,
Some(&f.attrs),
Some(&mut content),
);
let fed = match feed.take() {
None => Ok(()),
Some((handle, reader)) => {
drop(reader);
match handle.join() {
Ok(fed) => fed,
Err(panic) => std::panic::resume_unwind(panic),
}
}
};
match (made, fed) {
(Ok(made), Ok(())) => Ok(made),
(Err(e), _) | (Ok(_), Err(e)) => Err(e),
}
})?;
debug_assert_eq!(made.generation.get(), f.attrs.generation, "{}", f.name);
if made.map.len() != f.data_size {
return Err(crate::store::corrupt());
}
return Ok(f.data_size);
}
let made = self.provision2(&f.name, f.kind, opts, None, Some(&f.attrs), None)?;
debug_assert_eq!(made.generation.get(), f.attrs.generation, "{}", f.name);
debug_assert_eq!(made.map.len(), f.data_size, "{}", f.name);
if let Some((ranges, _)) = &f.full {
let pace = self.inner.domain.pace();
let piece = pace.map_or(CHUNK, |p| p.piece().min(CHUNK));
let mut buf = vec![0u8; piece];
let mut at = 0u64;
for &(file, len) in ranges {
let mut done = 0u64;
while done < len {
let n = (len - done).min(piece as u64).min(made.map.run_left(at)) as usize;
src.read_raw(&f.source, at, file + done, &mut buf[..n], reading)?;
let charge = pace.map(crate::pace::Pace::admit).transpose()?;
match f.kind {
RegionKind::Page => {
let to = self.page_region(&f.name)?;
let _ticket = to.write(to.pos(at)?, &buf[..n])?;
}
_ => {
let _ticket = self.append_region(&f.name)?.append(&buf[..n])?;
}
}
drop(charge);
done += n as u64;
at += n as u64;
}
}
return Ok(f.data_size);
}
match f.kind {
RegionKind::Slot => {
let Some(object) = &f.object else {
return Ok(0);
};
let _receipt = self.slot(&f.name)?.commit(object)?;
Ok(object.len() as u64)
}
RegionKind::Page | RegionKind::Append => {
if f.data_size != 0 {
return Err(crate::store::corrupt());
}
Ok(0)
}
}
}
}
#[cfg(all(test, not(loom)))]
mod tests {
#![allow(clippy::unwrap_used, unused_results)]
use std::path::Path;
use std::sync::{Arc, Mutex};
use store_io_core::IoClass;
use store_io_sim::world::TraceEvent;
use store_io_sim::{SimConfig, SimPlatform};
use crate::store::container_name;
use crate::{Store, StoreOptions};
fn background(p: &SimPlatform, dir: &str) -> Store<SimPlatform> {
Store::create_named(
p.clone(),
Path::new(dir),
StoreOptions::default(),
&container_name().unwrap(),
IoClass::Background,
)
.unwrap()
}
#[test]
fn test_a_copy_s_reader_shares_its_call_s_record_on_one_group() {
let p = SimPlatform::new(SimConfig::volatile(54));
let src = Store::create(p.clone(), Path::new("/a"), StoreOptions::default()).unwrap();
let dst = background(&p, "/c");
let own = dst.inner.domain.pace().unwrap();
let reader = dst.reader_pace(&src).unwrap();
assert!(reader.same_call(own));
assert!(
reader.is_second() && !own.is_second(),
"it leaves the fill a piece"
);
assert!(src.reader_pace(&dst).is_none());
}
#[test]
fn test_a_paced_raw_read_reads_charged_capped_pieces() {
let p = SimPlatform::new(SimConfig::volatile(55));
let s = Store::create(p.clone(), Path::new("/s"), StoreOptions::default()).unwrap();
let log = s.provision_append_region("log", 1 << 20).unwrap();
let data: Vec<u8> = (0..1usize << 20)
.map(|i| (i as u8) ^ ((i / 4096) as u8).wrapping_mul(37))
.collect();
let t = log.append(&data).unwrap();
let _receipt = log.sync_through(&t).unwrap();
let _hold = s.commit_hold();
let r = Arc::clone(log.state());
assert!(r.map.run_left(0) >= 1 << 20, "one run");
let at = r.map.locate(0, 1 << 20);
let pace = s.inner.domain.new_pace().unwrap();
let (s2, seen) = (s.clone(), Arc::new(Mutex::new(Vec::new())));
let out = Arc::clone(&seen);
p.with_world(|w| {
w.set_submit_probe(Some(Arc::new(move || {
let held: u64 = s2
.inner
.pool
.background_holdings()
.iter()
.map(|c| u64::from(c.1))
.sum();
out.lock().unwrap().push((s2.qos().in_flight, held));
})))
});
let from = p.with_world(|w| w.trace().len());
let mut back = vec![0u8; 1 << 20];
s.read_raw(&r, 0, at, &mut back, Some(&pace)).unwrap();
p.with_world(|w| w.set_submit_probe(None));
assert!(back == data);
let lens: Vec<u64> = p.with_world(|w| {
w.trace()[from..]
.iter()
.filter_map(|e| match e {
TraceEvent::Read { len, .. } => Some(*len),
_ => None,
})
.collect()
});
assert_eq!(lens, vec![256 << 10; 4]);
assert_eq!(*seen.lock().unwrap(), vec![(1, 1); 4]);
}
#[test]
fn test_an_unpaced_feed_sends_pieces_at_their_offsets() {
let p = SimPlatform::new(SimConfig::volatile(59));
let opts = StoreOptions {
max_io: 4 << 20,
..StoreOptions::default()
};
let s = Store::create(p.clone(), Path::new("/s"), opts).unwrap();
assert!(s.chunk() > super::FEED_PIECE, "a chunk of several parts");
let len = 8usize << 20;
let log = s.provision_append_region("log", len as u64).unwrap();
let data: Vec<u8> = (0..len)
.map(|i| (i as u8) ^ ((i / 4096) as u8).wrapping_mul(37) ^ ((i >> 20) as u8))
.collect();
let t = log.append(&data).unwrap();
let _receipt = log.sync_through(&t).unwrap();
let r = Arc::clone(log.state());
let back = std::thread::scope(|sc| {
let (reader, mut feed) = super::Feed::start(sc, &s, &r, len as u64, None).unwrap();
let mut back = vec![0u8; len];
for (i, part) in back.chunks_mut(1 << 20).enumerate() {
feed.read((i << 20) as u64, part).unwrap();
}
drop(feed);
reader.join().unwrap().unwrap();
back
});
assert!(back == data);
}
}
#[cfg(all(test, not(loom)))]
mod plan_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, unused_results)]
use std::path::Path;
use store_io_core::error::{Error, NotWrittenCause};
use store_io_format::slot::{FORMAT_1, FORMAT_2};
use store_io_sim::{SimConfig, SimPlatform};
use crate::test_hook::{Step, lib_tests};
use crate::{Store, StoreOptions};
type Action = Box<dyn FnOnce() + Send>;
fn store(seed: u64, format: u16) -> Store<SimPlatform> {
let p = SimPlatform::new(SimConfig::volatile(seed));
let opts = StoreOptions {
container_format: format,
..StoreOptions::default()
};
let s = Store::create(p, Path::new("/s"), opts).unwrap();
let _a = s.provision_append_region("a", 1 << 20).unwrap();
let _p = s.provision_page_region("p", 1 << 20).unwrap();
let slot = s.provision_slot("s").unwrap();
let _r = slot.commit(b"object").unwrap();
s
}
fn plan_with(s: &Store<SimPlatform>, act: Action) -> Result<super::CopyPlan, Error> {
lib_tests::at(Step::PlanHeaders, act);
let plan = s.copy_plan(false);
lib_tests::clear();
plan
}
fn stale(r: &Result<super::CopyPlan, Error>, id: Option<store_io_core::id::RegionId>) -> bool {
matches!(
r,
Err(Error::NotWritten {
cause: NotWrittenCause::StaleGeneration,
ctx,
}) if ctx.region.is_some() && (id.is_none() || ctx.region == id)
)
}
fn id_of(s: &Store<SimPlatform>, name: &str) -> Option<store_io_core::id::RegionId> {
s.regions_snapshot()
.iter()
.find(|r| r.name.as_str() == name)
.map(|r| r.id)
}
#[test]
fn test_a_plan_inside_a_recycle_s_window_fails() {
for (seed, format) in [(84, FORMAT_1), (85, FORMAT_2)] {
let s = store(seed, format);
let p_id = id_of(&s, "p");
let s2 = s.clone();
let seen = std::sync::Arc::new(std::sync::Mutex::new(None));
let out = std::sync::Arc::clone(&seen);
lib_tests::at(
Step::RecycleRewritten,
Box::new(move || {
let plan = s2.copy_plan(false);
*out.lock().unwrap() = Some(stale(&plan, p_id));
}),
);
let _q = s.page_region("p").unwrap().recycle().unwrap();
lib_tests::clear();
assert_eq!(*seen.lock().unwrap(), Some(true), "format {format}");
assert!(s.copy_plan(false).is_ok());
}
}
#[test]
fn test_a_region_changed_between_the_passes_fails_the_plan() {
for (seed, format) in [(80, FORMAT_1), (81, FORMAT_2)] {
let s = store(seed, format);
let plan = plan_with(&s, Box::new(|| {})).unwrap();
assert_eq!(plan.facts.len(), 3, "format {format}");
let slot = plan.facts.iter().find(|f| f.name == "s").unwrap();
assert_eq!(slot.object.as_deref(), Some(&b"object"[..]));
let s2 = s.clone();
let r = plan_with(
&s,
Box::new(move || {
let _n = s2.provision_append_region("new", 1 << 20).unwrap();
}),
);
assert!(
stale(&r, None),
"provisioned, format {format}: {:?}",
r.err()
);
let s2 = s.clone();
let r = plan_with(
&s,
Box::new(move || {
let _q = s2.page_region("p").unwrap().recycle().unwrap();
}),
);
assert!(
stale(&r, id_of(&s, "p")),
"recycled, format {format}: {:?}",
r.err()
);
let a_id = id_of(&s, "a");
let s2 = s.clone();
let r = plan_with(
&s,
Box::new(move || {
s2.append_region("a").unwrap().release().unwrap();
let _a = s2.provision_append_region("a", 1 << 20).unwrap();
}),
);
assert!(
stale(&r, a_id),
"released and again, format {format}: {:?}",
r.err()
);
let a_id = id_of(&s, "a");
let s2 = s.clone();
let r = plan_with(
&s,
Box::new(move || {
s2.append_region("a").unwrap().release().unwrap();
}),
);
assert!(stale(&r, a_id), "released, format {format}: {:?}", r.err());
assert!(s.copy_plan(false).is_ok(), "unchanged since");
}
}
#[test]
fn test_plans_racing_metadata_operations_never_deadlock() {
let (tx, rx) = std::sync::mpsc::channel();
let _w = std::thread::spawn(move || {
for (seed, format) in [(82, FORMAT_1), (83, FORMAT_2)] {
let s = store(seed, format);
let s2 = s.clone();
let ops = std::thread::spawn(move || {
for i in 0..60u64 {
let name = format!("r{}", i % 4);
if let Ok(r) = s2.provision_page_region(&name, 1 << 16) {
let _g = r.grow(1 << 17);
if i % 2 == 0 {
let _q = r.recycle();
}
let _r = r.release();
}
let _q = s2.page_region("p").and_then(|p| p.recycle());
}
});
let mut planned = 0;
while !ops.is_finished() || planned == 0 {
match s.copy_plan(false) {
Ok(_) => planned += 1,
Err(e) => assert!(
matches!(
e,
Error::NotWritten {
cause: NotWrittenCause::StaleGeneration,
..
}
),
"format {format}: {e:?}"
),
}
}
ops.join().unwrap();
}
let _sent = tx.send(());
});
rx.recv_timeout(std::time::Duration::from_secs(120))
.expect("a plan deadlocked against a metadata operation (or panicked)");
}
}