#![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)]
use std::convert::Infallible;
use std::marker::PhantomData;
use std::ptr::NonNull;
use crate::guard::{retire, Retired};
use crate::raw::node::{Node16, Node256, Node4, Node48, NodeHeader, NodeType};
use crate::raw::slot::TaggedPtr;
use crate::raw::{Layout, LeafNode, Storage};
use crate::sync::atomic::AtomicPtr;
pub(crate) struct HeapStorage<L> {
_marker: Invariant<L>,
}
pub(crate) type Invariant<L> = PhantomData<(L, fn(L) -> L)>;
impl<L> HeapStorage<L> {
pub(crate) const fn new() -> Self {
Self {
_marker: PhantomData,
}
}
}
unsafe impl<L: Send + Sync> Send for HeapStorage<L> {}
unsafe impl<L: Send + Sync> Sync for HeapStorage<L> {}
type A = AtomicPtr<u8>;
fn boxed<N>(n: N) -> NonNull<NodeHeader<A>> {
crate::raw::boxed(n).cast()
}
unsafe impl<L> Layout for HeapStorage<L> {
type Atomic = A;
type Leaf = L;
type Full = Infallible;
#[inline(always)]
unsafe fn node(&self, raw: TaggedPtr) -> NonNull<NodeHeader<A>> {
unsafe { NonNull::new_unchecked(raw.as_inner_ptr()) }
}
#[inline(always)]
unsafe fn leaf(&self, raw: TaggedPtr) -> NonNull<L> {
unsafe { NonNull::new_unchecked(raw.as_leaf_ptr()) }
}
#[inline(always)]
fn node_raw(&self, n: NonNull<NodeHeader<A>>) -> TaggedPtr {
TaggedPtr::from_inner(n.as_ptr())
}
#[inline(always)]
fn leaf_raw(&self, l: NonNull<L>) -> TaggedPtr {
TaggedPtr::from_leaf(l.as_ptr())
}
fn alloc_node(&self, ty: NodeType) -> Result<NonNull<NodeHeader<A>>, Infallible> {
#[cfg(test)]
NODE_COUNTS.with(|c| c.allocated.set(c.allocated.get() + 1));
Ok(match ty {
NodeType::Node4 => boxed(Node4::<A>::new()),
NodeType::Node16 => boxed(Node16::<A>::new()),
NodeType::Node48 => boxed(Node48::<A>::new()),
NodeType::Node256 => boxed(Node256::<A>::new()),
})
}
unsafe fn free_node(&self, n: NonNull<NodeHeader<A>>) {
#[cfg(test)]
NODE_COUNTS.with(|c| c.freed.set(c.freed.get() + 1));
unsafe {
match n.as_ref().node_type {
NodeType::Node4 => drop(Box::from_raw(n.cast::<Node4<A>>().as_ptr())),
NodeType::Node16 => drop(Box::from_raw(n.cast::<Node16<A>>().as_ptr())),
NodeType::Node48 => drop(Box::from_raw(n.cast::<Node48<A>>().as_ptr())),
NodeType::Node256 => drop(Box::from_raw(n.cast::<Node256<A>>().as_ptr())),
}
}
}
unsafe fn free_leaf(&self, l: NonNull<L>) {
drop(unsafe { Box::from_raw(l.as_ptr()) });
}
}
unsafe impl<L: LeafNode + Send + 'static> Storage for HeapStorage<L> {
type Guard = crossbeam_epoch::Guard;
unsafe fn retire_node(&self, n: NonNull<NodeHeader<A>>, guard: &crossbeam_epoch::Guard) {
#[cfg(test)]
RETIRED_NODES.with(|c| c.set(c.get() + 1));
#[cfg(test)]
NODE_COUNTS.with(|c| c.retired.set(c.retired.get() + 1));
unsafe {
match n.as_ref().node_type {
NodeType::Node4 => retire(guard, Retired::from_non_null(n.cast::<Node4<A>>())),
NodeType::Node16 => retire(guard, Retired::from_non_null(n.cast::<Node16<A>>())),
NodeType::Node48 => retire(guard, Retired::from_non_null(n.cast::<Node48<A>>())),
NodeType::Node256 => retire(guard, Retired::from_non_null(n.cast::<Node256<A>>())),
}
}
}
unsafe fn retire_leaf(&self, l: NonNull<L>, guard: &crossbeam_epoch::Guard) {
retire(guard, unsafe { Retired::from_non_null(l) });
}
}
#[cfg(test)]
#[derive(Default)]
pub(crate) struct NodeCounts {
pub(crate) allocated: std::cell::Cell<usize>,
pub(crate) freed: std::cell::Cell<usize>,
pub(crate) retired: std::cell::Cell<usize>,
}
#[cfg(test)]
std::thread_local! {
pub(crate) static RETIRED_NODES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
pub(crate) static NODE_COUNTS: NodeCounts = NodeCounts::default();
}