use std::sync::PoisonError;
use store_io_core::align::align_up;
use store_io_core::error::{Capability, Error, NotWrittenCause, Op, ReserveTag};
use store_io_format::judge::Standing;
use store_io_format::meta::{FillPattern, RegionState};
use store_io_format::meta2::{
FLAG_CAPPED, PAGE_HEADER_LEN, PageHeader, RawEntry, decode_region_header, encode_page,
max_extents,
};
use store_io_format::slot::FORMAT_2;
use store_io_platform::Platform;
use crate::io::{ctx, not_written};
use crate::layout::DATA_ALIGN;
use crate::provision2::Holds;
use crate::slots::PairRead;
use crate::space::lock_acct;
use crate::store::*;
const MIN_EXTENT: u64 = 1 << 20;
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub(crate) fn grow_region2(&self, r: &LiveRegion, new_size: u64) -> Result<u64, Error> {
let inner = &*self.inner;
let c = ctx(r);
if inner.layout.format != FORMAT_2 {
return Err(Error::Unsupported {
what: Capability::FormatVersion,
});
}
self.ready_for(r)?;
if inner.frozen.load(std::sync::atomic::Ordering::Acquire) {
return Err(not_written(NotWrittenCause::Frozen, c));
}
let _entry = entry_of(&lock_meta(&inner.meta), r)?;
let block = inner.layout.block();
let too_large = || not_written(NotWrittenCause::TooLarge, c);
let new = align_up(new_size, block).ok_or_else(too_large)?;
let mut header = r.header.lock().unwrap_or_else(PoisonError::into_inner);
let len = r.map.len();
if new <= len {
return Ok(len);
}
if new / block >= u64::from(u32::MAX) {
return Err(too_large());
}
let (fill, key) = header
.payload
.as_deref()
.and_then(|b| decode_region_header(b, inner.layout.log2_block, u64::MAX).ok())
.map(|v| (v.header.fill, v.header.fill_key))
.ok_or_else(corrupt)?;
if new > r.map.capacity() {
return self.grow_beyond2(r, &mut header, new, fill, &key);
}
self.fill_pieces(&r.map.pieces(len, new), fill, &key)?;
let mut meta = lock_meta(&inner.meta);
let (p, payload, old) = page_with(&meta, r, |e| {
debug_assert_eq!(e.data_size, len, "the map's length is the entry's");
e.data_size = new;
})?;
let meta_seq = crate::v2::next_meta_seq_of(&meta)?;
self.commit_page(&mut meta, p, &payload, meta_seq)?;
self.count_growth(&mut meta, &old, new, 0);
drop(meta);
r.map.publish_len(new);
if let Some(f) = &r.frontier {
f.raise_limit(new);
}
drop(header);
Ok(new)
}
fn grow_beyond2(
&self,
r: &LiveRegion,
header: &mut PairRead,
new: u64,
fill: FillPattern,
key: &[u8; 16],
) -> Result<u64, Error> {
let inner = &*self.inner;
let c = ctx(r);
let block = inner.layout.block();
let too_large = || not_written(NotWrittenCause::TooLarge, c);
let no_space = |tag| Error::NoSpace { tag, ctx: c };
let (len, old_cap) = (r.map.len(), r.map.capacity());
let mut meta = lock_meta(&inner.meta);
let (_, old) = entry_of(&meta, r)?;
debug_assert_eq!(
(old.data_size, old.capacity),
(len, old_cap),
"the map is the entry's"
);
if old.flags & FLAG_CAPPED != 0 {
return Err(no_space(ReserveTag::RegionLimit));
}
if usize::from(old.extent_count) >= max_extents(inner.layout.log2_block) {
return Err(no_space(ReserveTag::Extents));
}
let ext_len = align_up(MIN_EXTENT.max(old_cap / 8).max(new - old_cap), block)
.ok_or_else(too_large)?;
if !lock_acct(&inner.acct).fits(old.tag, ext_len) {
return Err(no_space(ReserveTag::Caller(old.tag)));
}
let last_end = r.map.extents().last().map_or(0, |(f, n)| f + n);
let reserved = lock_acct(&inner.acct).reserved();
let mut align = DATA_ALIGN;
let at = loop {
let v2 = meta.v2.as_deref().ok_or_else(corrupt)?;
let at = if v2.free.fits_at(last_end, ext_len) || last_end == v2.free.end() {
last_end
} else if let Some(at) = v2.free.first_fit(ext_len, align, 0) {
at
} else {
v2.free.tail(align, 0).ok_or_else(too_large)?
};
let end = at.checked_add(ext_len).ok_or_else(too_large)?;
let target = self
.tail(&meta)
.max(end)
.checked_add(reserved)
.ok_or_else(too_large)?;
match self.grow_to(&mut meta, target, old.tag) {
Ok(()) => break at,
Err(Error::NoSpace { .. }) if align > block => align = block,
Err(e) => return Err(growth_space(e)),
}
};
let capacity = old_cap.checked_add(ext_len).ok_or_else(too_large)?;
let container = meta.container_len;
if let Some(v2) = meta.v2.as_deref_mut() {
v2.mark(&[(at, ext_len)], container);
}
lock_acct(&inner.acct).add_usage(old.tag, 0, ext_len);
drop(meta);
let mut holds = Holds {
store: self,
ranges: vec![(at, ext_len)],
claim: None,
tag: old.tag,
charged: ext_len,
from: None,
armed: true,
};
let grown = Beyond {
new,
fill,
key,
old,
at,
ext_len,
capacity,
len,
old_cap,
};
self.fill_and_commit_beyond2(r, header, &grown, &mut holds)?;
r.map.extend(at, ext_len);
r.map.publish_len(new);
if let Some(f) = &r.frontier {
f.raise_limit(new);
}
Ok(new)
}
fn fill_and_commit_beyond2(
&self,
r: &LiveRegion,
header: &mut PairRead,
g: &Beyond<'_>,
holds: &mut Holds<'_, P>,
) -> Result<(), Error> {
let inner = &*self.inner;
let (at, ext_len) = (g.at, g.ext_len);
let filled = if self.writes_slack() {
ext_len
} else {
g.new - g.old_cap
};
if filled < ext_len {
inner
.platform
.allocate_range(&inner.file, at + filled, ext_len - filled)
.map_err(|raw| growth_space(space_err(Op::Allocate, raw, g.old.tag)))?;
}
let mut pieces = r.map.pieces(g.len, g.old_cap);
pieces.push((at, g.old_cap, filled));
self.fill_pieces(&pieces, g.fill, g.key)?;
let committed = usize::from(g.old.extent_count);
let listed = header
.payload
.as_deref()
.and_then(|b| decode_region_header(b, inner.layout.log2_block, u64::MAX).ok())
.map_or(0, |v| usize::from(v.header.extent_count));
if listed > committed {
let _same = self.rewrite_header2(r, header, RegionState::Ready, 0, committed, None)?;
}
let _same = self.rewrite_header2(
r,
header,
RegionState::Ready,
0,
committed,
Some((at, ext_len)),
)?;
let mut meta = lock_meta(&inner.meta);
let meta_seq = crate::v2::next_meta_seq_of(&meta)?;
let (p, payload, _) = page_with(&meta, r, |e| {
e.data_size = g.new;
e.capacity = g.capacity;
e.extent_count += 1;
e.span_start = e.span_start.min(at);
e.span_end = e.span_end.max(at + ext_len);
e.footprint_seq = meta_seq;
})?;
self.commit_page(&mut meta, p, &payload, meta_seq)?;
holds.armed = false;
if let Some(v2) = meta.v2.as_deref_mut() {
v2.settle(ext_len);
}
self.count_growth(&mut meta, &g.old, g.new, ext_len);
Ok(())
}
fn fill_pieces(
&self,
pieces: &[(u64, u64, u64)],
fill: FillPattern,
key: &[u8; 16],
) -> Result<(), Error> {
for &(file, at, n) in pieces {
self.fill_range(file, n, 0, fill, key, at)
.map_err(growth_space)?;
}
self.flush_all_accounted()?;
for &(file, _, n) in pieces {
self.verify_ready(file, n)?;
}
Ok(())
}
fn commit_page(
&self,
meta: &mut Meta,
p: usize,
payload: &[u8],
meta_seq: u64,
) -> Result<(), Error> {
let inner = &*self.inner;
let mut q = inner.queues.get(0);
crate::v2::write_page::<P>(
&mut q,
&inner.file,
&inner.pool,
&inner.layout,
inner.volume,
&inner.policy,
&inner.domain,
meta,
p,
payload,
meta_seq,
inner.owner_generation,
)
.inspect_err(|_| self.wake_frontiers())
}
fn count_growth(&self, meta: &mut Meta, old: &RawEntry, new: u64, extra: u64) {
let physical = old.capacity.saturating_add(2 * self.inner.layout.block());
if let Some(v2) = meta.v2.as_deref_mut() {
let u = v2
.usage
.iter_mut()
.find(|u| **u == (old.tag, old.data_size, physical));
debug_assert!(u.is_some(), "a usable region without its usage");
if let Some(u) = u {
*u = (old.tag, new, physical.saturating_add(extra));
}
}
let mut acct = lock_acct(&self.inner.acct);
acct.remove_usage(old.tag, old.data_size, 0);
acct.add_usage(old.tag, new, 0);
}
}
fn entry_of(meta: &Meta, r: &LiveRegion) -> Result<(usize, RawEntry), Error> {
let id = r.id.get();
let seq = (id & 0xFFFF_FFFF) as u32;
let p = usize::try_from(id >> 32).map_err(|_| corrupt())?;
let v2 = meta.v2.as_deref().ok_or_else(corrupt)?;
let page = v2.pages.get(p).ok_or_else(corrupt)?;
if page.standing != Standing::Confirmed || page.ahead().next().is_some() {
return Err(not_written(NotWrittenCause::Frozen, ctx(r)));
}
let e = page
.entries
.iter()
.find(|e| e.seq == seq)
.ok_or_else(corrupt)?;
Ok((p, *e))
}
fn page_with(
meta: &Meta,
r: &LiveRegion,
change: impl FnOnce(&mut RawEntry),
) -> Result<(usize, Vec<u8>, RawEntry), Error> {
let (p, old) = entry_of(meta, r)?;
let v2 = meta.v2.as_deref().ok_or_else(corrupt)?;
let page = &v2.pages[p];
let ph = page.header.ok_or_else(corrupt)?;
let mut raws = page.entries.clone();
let e = raws
.iter_mut()
.find(|e| e.seq == old.seq)
.ok_or_else(corrupt)?;
change(e);
let next = PageHeader {
page_index: ph.page_index,
next_seq: ph.next_seq,
count: raws.len() as u16,
};
let mut buf = vec![0u8; PAGE_HEADER_LEN + 96 * raws.len()];
let n = encode_page(&mut buf, &next, &raws).map_err(|_| meta_err())?;
buf.truncate(n);
Ok((p, buf, old))
}
fn growth_space(e: Error) -> Error {
match e {
Error::NoSpace { ctx, .. } => Error::NoSpace {
tag: ReserveTag::Growth,
ctx,
},
e => e,
}
}
struct Beyond<'a> {
new: u64,
fill: FillPattern,
key: &'a [u8; 16],
old: RawEntry,
at: u64,
ext_len: u64,
capacity: u64,
len: u64,
old_cap: u64,
}