use core::alloc::{Layout, LayoutError};
use core::convert::{TryFrom, TryInto};
use core::fmt::Debug;
use core::hash::Hash;
use core::marker::PhantomData;
use core::mem::MaybeUninit;
use core::ops::{Range, RangeBounds};
use core::ptr::NonNull;
pub unsafe trait Capacity: Copy + Debug + Eq + Hash + Ord {
const MAX_REPRESENTABLE: usize;
fn from_usize(i: usize) -> Self;
fn as_usize(&self) -> usize;
}
#[cold]
#[inline(never)]
#[track_caller]
pub(crate) fn buffer_too_large_for_index_type<I: Capacity>() {
panic!(
"provided storage block cannot be fully indexed by type {}",
core::any::type_name::<I>()
);
}
pub(crate) fn normalize_range<I: Capacity, R: RangeBounds<I>>(range: R, max_end: usize) -> Range<usize> {
use core::ops::Bound;
let start = match range.start_bound() {
Bound::Included(x) => x.as_usize(),
Bound::Excluded(x) => x.as_usize().saturating_add(1),
Bound::Unbounded => 0,
};
let end = match range.end_bound() {
Bound::Included(x) => x.as_usize().saturating_add(1),
Bound::Excluded(x) => x.as_usize(),
Bound::Unbounded => max_end,
};
assert!(end <= max_end, "invalid range specifier: end (is {:?}) is greater than {:?}", end, max_end);
assert!(start <= end, "invalid range specifier: start (is {:?}) is greater than end (is {:?})", start, end);
Range { start, end }
}
#[allow(clippy::cast_possible_truncation)]
unsafe impl Capacity for u8 {
const MAX_REPRESENTABLE: usize = 0xFF;
#[inline]
fn from_usize(i: usize) -> Self {
debug_assert!(<usize as TryInto<Self>>::try_into(i).is_ok());
i as u8
}
#[inline]
fn as_usize(&self) -> usize {
*self as usize
}
}
#[allow(clippy::cast_possible_truncation)]
unsafe impl Capacity for u16 {
const MAX_REPRESENTABLE: usize = 0xFFFF;
#[inline]
fn from_usize(i: usize) -> Self {
debug_assert!(<usize as TryInto<Self>>::try_into(i).is_ok());
i as u16
}
#[inline]
fn as_usize(&self) -> usize {
*self as usize
}
}
#[allow(clippy::cast_possible_truncation)]
unsafe impl Capacity for u32 {
const MAX_REPRESENTABLE: usize = 0xFFFF_FFFF;
#[inline]
fn from_usize(i: usize) -> Self {
debug_assert!(<usize as TryInto<Self>>::try_into(i).is_ok());
i as u32
}
#[inline]
fn as_usize(&self) -> usize {
debug_assert!(<usize as TryFrom<Self>>::try_from(*self).is_ok());
*self as usize
}
}
#[allow(clippy::cast_possible_truncation)]
unsafe impl Capacity for u64 {
const MAX_REPRESENTABLE: usize = usize::max_value();
#[inline]
fn from_usize(i: usize) -> Self {
debug_assert!(<usize as TryInto<Self>>::try_into(i).is_ok());
i as u64
}
#[inline]
fn as_usize(&self) -> usize {
debug_assert!(<usize as TryFrom<Self>>::try_from(*self).is_ok());
*self as usize
}
}
unsafe impl Capacity for usize {
const MAX_REPRESENTABLE: usize = usize::max_value();
#[inline]
fn from_usize(i: usize) -> Self {
i
}
#[inline]
fn as_usize(&self) -> usize {
*self
}
}
#[macro_export]
macro_rules! index_type {
( $(#[$attrs:meta])* $v:vis $name:ident: $repr:ty ) => {
$(#[$attrs])*
#[derive(
core::marker::Copy,
core::clone::Clone,
core::default::Default,
core::fmt::Debug,
core::hash::Hash,
core::cmp::PartialEq,
core::cmp::Eq,
core::cmp::PartialOrd,
core::cmp::Ord)]
$v struct $name($repr);
unsafe impl $crate::storage::Capacity for $name {
const MAX_REPRESENTABLE: usize = <$repr as $crate::storage::Capacity>::MAX_REPRESENTABLE;
#[inline]
#[track_caller]
fn from_usize(i: usize) -> Self {
Self(<$repr as $crate::storage::Capacity>::from_usize(i))
}
#[inline]
#[track_caller]
fn as_usize(&self) -> usize {
<$repr as $crate::storage::Capacity>::as_usize(&self.0)
}
}
};
( $(#[$attrs:meta])* $v:vis $name:ident: $repr:ty ; $($rest:tt)* ) => {
$crate::index_type!($(#[$attrs])* $v $name: $repr);
$crate::index_type!($($rest)*);
};
() => {}
}
pub trait LayoutSpec {
fn layout_with_capacity(items: usize) -> Result<Layout, LayoutError>;
}
pub struct ArrayLayout<T>(PhantomData<T>);
impl<T> LayoutSpec for ArrayLayout<T> {
fn layout_with_capacity(items: usize) -> Result<Layout, LayoutError> {
Layout::array::<T>(items)
}
}
#[allow(clippy::missing_safety_doc)] pub unsafe trait Storage<R: LayoutSpec>: Sized {
fn get_ptr(&self) -> *const u8;
fn get_mut_ptr(&mut self) -> *mut u8;
fn capacity(&self) -> usize;
}
#[inline(always)]
pub(crate) fn ptr_at_index<T, S: Storage<ArrayLayout<T>>>(storage: &S, index: usize) -> *const T {
debug_assert!(index <= storage.capacity());
let ptr = storage.get_ptr().cast::<T>();
ptr.wrapping_add(index)
}
#[inline(always)]
pub(crate) fn mut_ptr_at_index<T, S: Storage<ArrayLayout<T>>>(
storage: &mut S,
index: usize,
) -> *mut T {
debug_assert!(index <= storage.capacity());
let ptr = storage.get_mut_ptr().cast::<T>();
ptr.wrapping_add(index)
}
pub type SliceStorage<'a, T> = &'a mut [MaybeUninit<T>];
unsafe impl<T: Sized> Storage<ArrayLayout<T>> for &mut [MaybeUninit<T>] {
#[inline]
fn get_ptr(&self) -> *const u8 {
self.as_ptr().cast()
}
#[inline]
fn get_mut_ptr(&mut self) -> *mut u8 {
self.as_mut_ptr().cast()
}
#[inline]
fn capacity(&self) -> usize {
self.len()
}
}
pub struct ArenaStorage<'src, R: LayoutSpec> {
ptr: NonNull<u8>,
cap: usize,
spec: PhantomData<R>,
src: PhantomData<&'src ()>,
}
impl<R: LayoutSpec> ArenaStorage<'_, R> {
pub(crate) unsafe fn from_raw_parts(ptr: *mut u8, cap: usize) -> Option<Self> {
let ptr = NonNull::new(ptr)?;
Some(ArenaStorage {
ptr,
cap,
spec: PhantomData,
src: PhantomData,
})
}
}
unsafe impl<R: LayoutSpec> Storage<R> for ArenaStorage<'_, R> {
fn get_ptr(&self) -> *const u8 {
self.ptr.as_ptr() as _
}
fn get_mut_ptr(&mut self) -> *mut u8 {
self.ptr.as_ptr()
}
fn capacity(&self) -> usize {
self.cap
}
}
unsafe impl<R: LayoutSpec, S: Storage<R>> Storage<R> for crate::arena::Box<'_, S> {
#[inline]
fn get_ptr(&self) -> *const u8 {
(**self).get_ptr()
}
#[inline]
fn get_mut_ptr(&mut self) -> *mut u8 {
(**self).get_mut_ptr()
}
#[inline]
fn capacity(&self) -> usize {
(**self).capacity()
}
}
unsafe impl<R: LayoutSpec, S: Storage<R>> Storage<R> for &mut S {
fn get_ptr(&self) -> *const u8 {
(**self).get_ptr()
}
fn get_mut_ptr(&mut self) -> *mut u8 {
(**self).get_mut_ptr()
}
fn capacity(&self) -> usize {
(**self).capacity()
}
}
#[cfg(feature = "alloc")]
#[cfg_attr(docs_rs, doc(cfg(feature = "alloc")))]
pub struct AllocStorage<R: LayoutSpec> {
ptr: NonNull<u8>,
cap: usize,
spec: PhantomData<R>,
}
#[cfg(feature = "alloc")]
#[cfg_attr(docs_rs, doc(cfg(feature = "alloc")))]
impl<R: LayoutSpec> AllocStorage<R> {
pub fn with_capacity(capacity: usize) -> Self {
let layout =
R::layout_with_capacity(capacity).expect("layout error in AllocStorage::with_capacity");
let ptr = unsafe { alloc::alloc::alloc(layout) };
let ptr = NonNull::new(ptr).expect("allocation failure in AllocStorage::with_capacity");
AllocStorage {
ptr,
cap: capacity,
spec: PhantomData,
}
}
}
#[cfg(feature = "alloc")]
#[cfg_attr(docs_rs, doc(cfg(feature = "alloc")))]
impl<R: LayoutSpec> Drop for AllocStorage<R> {
fn drop(&mut self) {
let layout = R::layout_with_capacity(self.cap).expect("dropped an invalid AllocStorage");
unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), layout) };
}
}
#[cfg(feature = "alloc")]
#[cfg_attr(docs_rs, doc(cfg(feature = "alloc")))]
unsafe impl<R: LayoutSpec> Storage<R> for AllocStorage<R> {
fn get_ptr(&self) -> *const u8 {
self.ptr.as_ptr() as _
}
fn get_mut_ptr(&mut self) -> *mut u8 {
self.ptr.as_ptr()
}
fn capacity(&self) -> usize {
self.cap
}
}
#[cfg(feature = "alloc")]
#[cfg_attr(docs_rs, doc(cfg(feature = "alloc")))]
unsafe impl<R: LayoutSpec, S: Storage<R>> Storage<R> for alloc::boxed::Box<S> {
fn get_ptr(&self) -> *const u8 {
(**self).get_ptr()
}
fn get_mut_ptr(&mut self) -> *mut u8 {
(**self).get_mut_ptr()
}
fn capacity(&self) -> usize {
(**self).capacity()
}
}
pub type InlineStorage<T, const C: usize> = [MaybeUninit<T>; C];
unsafe impl<T, const C: usize> Storage<ArrayLayout<T>> for InlineStorage<T, C> {
fn get_ptr(&self) -> *const u8 {
self.as_ptr().cast()
}
fn get_mut_ptr(&mut self) -> *mut u8 {
self.as_mut_ptr().cast()
}
fn capacity(&self) -> usize {
C
}
}