use std::io;
use bstack::{BStackGenOp, BStackRange};
use crate::BStackRaiiAllocator;
use crate::io_core::bulk::seq_free_many;
use crate::io_core::{WalStatus, WalTxn};
use crate::primitives::{Offset, checked_off};
use crate::types::compiled::owned::BStackOwned;
use crate::types::compiled::rc::{CTRL_WEAK_OFFSET, strong_counter_off};
use crate::types::compiled::vec::{BYTEVEC_HEADER, VecDesc};
use crate::types::traits::{BStackBlock, BStackRef, BStackShared};
use crate::util::io_error;
pub trait TryCloneIn: BStackBlock + Sized {
fn try_clone_in<A: BStackRaiiAllocator>(&self, allocator: &A) -> io::Result<BStackOwned<Self>> {
let mut plan = ClonePlan::new();
let dst = match self.__bstack_clone_into(allocator, &mut plan) {
Ok(range) => range,
Err(e) => {
plan.rollback(allocator);
return Err(e);
}
};
plan.commit(allocator)?;
Ok(unsafe { BStackOwned::from_raw(Self::from_range(dst)) })
}
}
pub struct ClonePlan {
mode: Mode,
allocated: Vec<BStackRange>,
sizes: Vec<u64>,
cursor: usize,
fake_next: u64,
writes: Vec<(u64, Vec<u8>)>,
bumps: Vec<u64>,
wal: Option<WalTxn>,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Mode {
Direct,
Measure,
Build,
}
impl Default for ClonePlan {
fn default() -> Self {
Self::new()
}
}
impl ClonePlan {
pub fn new() -> Self {
ClonePlan {
mode: Mode::Direct,
allocated: Vec::new(),
sizes: Vec::new(),
cursor: 0,
fake_next: 1,
writes: Vec::new(),
bumps: Vec::new(),
wal: None,
}
}
#[cfg(test)]
pub(crate) fn for_build_test(allocated: Vec<BStackRange>) -> Self {
let mut p = Self::new();
p.mode = Mode::Build;
p.allocated = allocated;
p
}
pub fn is_measuring(&self) -> bool {
matches!(self.mode, Mode::Measure)
}
pub fn alloc_raw<A: BStackRaiiAllocator>(
&mut self,
allocator: &A,
size: u64,
) -> io::Result<BStackRange> {
match self.mode {
Mode::Measure => {
self.sizes.push(size);
let off = self.fake_next;
self.fake_next = self.fake_next.saturating_add(size.max(1));
Ok(BStackRange::new(off, size))
}
Mode::Build => {
let range = *self.allocated.get(self.cursor).ok_or_else(|| {
io::Error::other(
"clone build/measure block-count mismatch (source mutated mid-clone?)",
)
})?;
self.cursor += 1;
if range.len() != size {
return Err(io::Error::other(
"clone build/measure size mismatch (source mutated mid-clone?)",
));
}
Ok(range)
}
Mode::Direct => {
crate::io_core::alloc_logged(allocator, &mut self.wal, &mut self.allocated, size)
}
}
}
pub fn write(&mut self, offset: u64, bytes: Vec<u8>) {
if self.mode != Mode::Measure {
self.writes.push((offset, bytes));
}
}
pub fn stage_bytevec<A: BStackRaiiAllocator>(
&mut self,
allocator: &A,
data: &[u8],
) -> io::Result<VecDesc> {
let len = data.len() as u64;
let size = BYTEVEC_HEADER + len;
let range = self.alloc_raw(allocator, size)?;
if self.mode != Mode::Measure {
self.write(
range.start(),
crate::types::compiled::vec::bytevec_image(len, len, data),
);
}
Ok(VecDesc {
data_off: Offset::from_raw(range.start()),
data_size: size,
})
}
pub fn bump_strong<T: BStackShared, A: BStackRaiiAllocator>(
&mut self,
data: BStackRef<T>,
allocator: &A,
) -> io::Result<()> {
if self.mode == Mode::Measure {
return Ok(());
}
let (data_ref, ctrl) = T::strong_parts(data, allocator)?;
let off = strong_counter_off(data_ref.into_range().start(), ctrl)?;
self.bumps.push(off);
Ok(())
}
pub fn bump_weak(&mut self, ctrl_off: u64) -> io::Result<()> {
if self.mode != Mode::Measure {
self.bumps.push(checked_off(ctrl_off, CTRL_WEAK_OFFSET)?);
}
Ok(())
}
pub fn run_clone<A, F>(allocator: &A, mut descend: F) -> io::Result<BStackRange>
where
A: BStackRaiiAllocator,
F: FnMut(&mut ClonePlan) -> io::Result<BStackRange>,
{
if allocator.atomic_bulk() {
let mut plan = ClonePlan::new();
plan.mode = Mode::Measure;
descend(&mut plan)?;
plan.allocate(allocator)?;
plan.mode = Mode::Build;
match descend(&mut plan) {
Ok(dst) => {
plan.commit(allocator)?;
Ok(dst)
}
Err(e) => {
plan.rollback(allocator);
Err(e)
}
}
} else {
let mut plan = ClonePlan::new();
match descend(&mut plan) {
Ok(dst) => {
plan.commit(allocator)?;
Ok(dst)
}
Err(e) => {
plan.rollback(allocator);
Err(e)
}
}
}
}
fn allocate<A: BStackRaiiAllocator>(&mut self, allocator: &A) -> io::Result<()> {
let ranges = allocator.alloc_many(&self.sizes)?;
if allocator.wal_anchor().is_some() && !ranges.is_empty() {
match WalTxn::begin_many(allocator, &ranges) {
Ok(txn) => self.wal = Some(txn),
Err(e) => {
let _ = unsafe { allocator.free_many(ranges) };
return Err(e);
}
}
}
self.allocated = ranges;
self.cursor = 0;
Ok(())
}
pub fn rollback<A: BStackRaiiAllocator>(self, allocator: &A) {
Self::reclaim(&self.wal, self.allocated, allocator);
}
pub fn commit<A: BStackRaiiAllocator>(self, allocator: &A) -> io::Result<()> {
let ClonePlan {
allocated,
writes,
mut bumps,
wal,
..
} = self;
let stack = allocator.stack();
bumps.sort_unstable();
let mut counters: Vec<(u64, u64)> = Vec::new();
for off in bumps {
match counters.last_mut() {
Some(last) if last.0 == off => last.1 += 1,
_ => counters.push((off, 1)),
}
}
let n = counters.len();
let mut readbufs: Vec<[u8; 8]> = vec![[0u8; 8]; n];
let mut newbufs: Vec<[u8; 8]> = vec![[0u8; 8]; n];
let mut overflow = false;
let mut read_err: Option<io::Error> = None;
let flip = [WalStatus::Complete as u8];
let flip_off = wal.as_ref().map(|w| w.commit_marker_off());
let mut flipped = flip_off.is_none();
let mut read_i = 0usize;
let mut computed = false;
let mut cwrite_i = 0usize;
let mut write_i = 0usize;
let result = stack.inplace_gen(|feedback| {
if let Err(e) = feedback {
read_err = Some(e);
return None; }
if read_i < n {
let i = read_i;
read_i += 1;
let buf: &mut [u8] =
unsafe { core::mem::transmute::<&mut [u8], _>(&mut readbufs[i][..]) };
return Some(BStackGenOp::Read {
offset: counters[i].0,
buf,
});
}
if !computed {
computed = true;
for i in 0..n {
let cur = u64::from_le_bytes(readbufs[i]);
match cur.checked_add(counters[i].1) {
Some(v) => newbufs[i] = v.to_le_bytes(),
None => overflow = true,
}
}
if overflow {
return None;
}
}
if cwrite_i < n {
let i = cwrite_i;
cwrite_i += 1;
let data: &[u8] = unsafe { core::mem::transmute::<&[u8], _>(&newbufs[i][..]) };
return Some(BStackGenOp::Write {
offset: counters[i].0,
data,
});
}
if write_i < writes.len() {
let i = write_i;
write_i += 1;
let (off, ref bytes) = writes[i];
let data: &[u8] = unsafe { core::mem::transmute::<&[u8], _>(bytes.as_slice()) };
return Some(BStackGenOp::Write { offset: off, data });
}
if !flipped {
flipped = true;
let data: &[u8] = unsafe { core::mem::transmute::<&[u8], _>(&flip[..]) };
return Some(BStackGenOp::Write {
offset: flip_off.unwrap(),
data,
});
}
None
});
match result {
Ok(()) if read_err.is_some() => {
Self::reclaim(&wal, allocated, allocator);
Err(read_err.unwrap())
}
Ok(()) if overflow => {
Self::reclaim(&wal, allocated, allocator);
Err(io_error!("refcount overflow while committing clone"))
}
Ok(()) => {
if let Some(w) = &wal {
let _ = w.set_idle(allocator);
}
Ok(())
}
Err(e) => {
Self::reclaim(&wal, allocated, allocator);
Err(e)
}
}
}
fn reclaim<A: BStackRaiiAllocator>(
wal: &Option<WalTxn>,
allocated: Vec<BStackRange>,
allocator: &A,
) {
match wal {
Some(w) => {
let _ = w.finish(allocator);
}
None => {
let _ = seq_free_many(allocator, allocated);
}
}
}
}