use std::ptr;
use allocator_api2::alloc::{Allocator, Layout};
use super::bitmask::BitMask;
use super::config::{CACHE_LINE, GROUP_SIZE};
use super::control::{CTRL_EMPTY, CTRL_TOMBSTONE, ControlByte};
use super::error::TryReserveError;
use super::iter::RegionIter;
use super::simd;
pub(crate) struct Arena {
ptr: ptr::NonNull<u8>,
layout: Layout,
}
impl Arena {
#[inline]
pub(crate) const fn empty() -> Self {
Self {
ptr: ptr::NonNull::dangling(),
layout: unsafe { Layout::from_size_align_unchecked(0, 1) },
}
}
pub(crate) fn try_allocate_with_ctrl_zeroed<A: Allocator>(
layout: Layout,
ctrl_bytes: usize,
alloc: &A,
) -> Result<Self, TryReserveError> {
if layout.size() == 0 {
return Ok(Self::empty());
}
let ptr = alloc
.allocate(layout)
.map_err(|_| TryReserveError::AllocError)?
.cast::<u8>();
if ctrl_bytes > 0 {
unsafe { ptr::write_bytes(ptr.as_ptr(), 0, ctrl_bytes) };
}
Ok(Self { ptr, layout })
}
#[inline]
pub(crate) fn as_ptr(&self) -> *mut u8 {
self.ptr.as_ptr()
}
pub(crate) fn deallocate<A: Allocator>(self, alloc: &A) {
if self.layout.size() == 0 {
return;
}
unsafe { alloc.deallocate(self.ptr, self.layout) };
}
}
pub(crate) fn layout_for<K, V>(total_ctrl: usize) -> Result<(Layout, usize), TryReserveError> {
let total_ctrl = total_ctrl.max(1);
let ctrl_layout =
Layout::from_size_align(total_ctrl, CACHE_LINE).map_err(|_| TryReserveError::AllocError)?;
let data_layout =
Layout::array::<SlotEntry<K, V>>(total_ctrl).map_err(|_| TryReserveError::AllocError)?;
let (arena_layout, data_base_off) = ctrl_layout
.extend(data_layout)
.map_err(|_| TryReserveError::AllocError)?;
Ok((arena_layout.pad_to_align(), data_base_off))
}
#[inline]
pub(crate) fn check_disjoint_aliasing<T: PartialEq, const N: usize>(locations: &[Option<T>; N]) {
for (i, li) in locations.iter().enumerate() {
let Some(li) = li else { continue };
for other in &locations[i + 1..] {
assert!(
other.as_ref() != Some(li),
"get_disjoint_mut: duplicate keys resolve to the same entry",
);
}
}
}
pub(crate) struct DeallocGuard<'a, A: Allocator> {
arena: Option<Arena>,
alloc: &'a A,
}
impl<'a, A: Allocator> DeallocGuard<'a, A> {
#[inline]
pub(crate) fn new(arena: Arena, alloc: &'a A) -> Self {
Self {
arena: Some(arena),
alloc,
}
}
}
impl<A: Allocator> Drop for DeallocGuard<'_, A> {
fn drop(&mut self) {
if let Some(arena) = self.arena.take() {
arena.deallocate(self.alloc);
}
}
}
pub(crate) struct SlotEntry<K, V> {
pub(crate) key: K,
pub(crate) value: V,
}
impl<K: Clone, V: Clone> Clone for SlotEntry<K, V> {
fn clone(&self) -> Self {
Self {
key: self.key.clone(),
value: self.value.clone(),
}
}
}
pub(crate) fn clone_region_panic_safe<K: Clone, V: Clone>(
src_ctrl: *const u8,
dst_ctrl: *mut u8,
src_slots: *const SlotEntry<K, V>,
dst_slots: *mut SlotEntry<K, V>,
capacity: usize,
) {
for idx in 0..capacity {
let ctrl = unsafe { *src_ctrl.add(idx) };
if ctrl.is_occupied() {
let cloned = unsafe { (*src_slots.add(idx)).clone() };
unsafe { dst_slots.add(idx).write(cloned) };
unsafe { *dst_ctrl.add(idx) = ctrl };
}
}
for idx in 0..capacity {
let ctrl = unsafe { *src_ctrl.add(idx) };
if ctrl == CTRL_TOMBSTONE {
unsafe { *dst_ctrl.add(idx) = CTRL_TOMBSTONE };
}
}
}
pub(crate) trait ArenaSlots<T> {
fn ctrl_ptr(&self) -> *mut u8;
fn data_ptr(&self) -> *mut T;
fn capacity(&self) -> usize;
#[inline]
fn group_ctrl(&self, group_idx: usize) -> *const u8 {
unsafe { self.ctrl_ptr().add(group_idx * GROUP_SIZE) }
}
#[inline]
fn control_at(&self, idx: usize) -> u8 {
unsafe { *self.ctrl_ptr().add(idx) }
}
#[inline]
fn set_control(&self, idx: usize, ctrl: u8) {
unsafe { *self.ctrl_ptr().add(idx) = ctrl }
}
#[inline]
fn mark_tombstone(&self, idx: usize) {
self.set_control(idx, CTRL_TOMBSTONE);
}
#[inline]
fn clear_all_controls(&self) {
if self.capacity() == 0 {
return;
}
unsafe { ptr::write_bytes(self.ctrl_ptr(), 0, self.capacity()) }
}
#[inline]
fn write_with_control(&self, idx: usize, entry: T, ctrl: u8) {
unsafe { self.data_ptr().add(idx).write(entry) }
self.set_control(idx, ctrl);
}
#[inline]
unsafe fn get_ref(&self, idx: usize) -> &T {
unsafe { &*self.data_ptr().add(idx) }
}
#[inline]
unsafe fn get_mut(&mut self, idx: usize) -> &mut T {
unsafe { &mut *self.data_ptr().add(idx) }
}
#[inline]
unsafe fn take(&self, idx: usize) -> T {
unsafe { self.data_ptr().add(idx).read() }
}
#[inline]
fn group_match_mask(&self, group_idx: usize, target: u8) -> BitMask {
unsafe { simd::eq_mask_16(self.group_ctrl(group_idx), target) }
}
#[inline]
fn group_free_mask(&self, group_idx: usize) -> BitMask {
unsafe { simd::free_mask_16(self.group_ctrl(group_idx)) }
}
#[inline]
fn first_free_in_group(&self, group_idx: usize) -> Option<usize> {
let offset = self.group_free_mask(group_idx).lowest()?;
let slot_idx = group_idx * GROUP_SIZE + offset;
if slot_idx < self.capacity() {
Some(slot_idx)
} else {
None
}
}
#[inline]
fn occupied(&self) -> RegionIter<'_, T, Self>
where
Self: Sized,
{
RegionIter::new(std::slice::from_ref(self))
}
fn drop_values(&self) {
if self.capacity() == 0 {
return;
}
let ctrl = self.ctrl_ptr();
let slots = self.data_ptr();
for idx in 0..self.capacity() {
if unsafe { (*ctrl.add(idx)).is_occupied() } {
unsafe { ptr::drop_in_place(slots.add(idx)) }
}
}
}
fn drop_values_and_clear(&self) {
if self.capacity() == 0 {
return;
}
let ctrl = self.ctrl_ptr();
let slots = self.data_ptr();
for idx in 0..self.capacity() {
unsafe {
let prev = *ctrl.add(idx);
*ctrl.add(idx) = CTRL_EMPTY;
if prev.is_occupied() {
ptr::drop_in_place(slots.add(idx));
}
}
}
}
}