use {
crate::{
error::{ArithErr, ArithOp, Cause, Error, LayoutErr},
layout::Layout,
traits::{
alloc::BasicAlloc,
data::type_props::{PtrProps, SizedProps, VarSized}
}
},
::core::{
clone::Clone,
iter::{IntoIterator, Iterator},
marker::{Copy, Sized},
mem::forget,
ops::{Deref, Drop, Fn},
option::Option::{self, None, Some},
ptr::{self, NonNull},
result::Result::{self, Err, Ok}
}
};
pub const USIZE_MAX_NO_HIGH_BIT: usize = usize::MAX >> 1;
pub const USIZE_HIGH_BIT: usize = usize::MAX ^ (USIZE_MAX_NO_HIGH_BIT);
#[::rustversion::attr(since(1.47), const)]
pub fn checked_op(l: usize, op: ArithOp, r: usize) -> Result<usize, ArithErr> {
#[::rustversion::since(1.52)]
#[inline(always)]
const fn checked_div(l: usize, r: usize) -> Option<usize> {
l.checked_div(r)
}
#[::rustversion::before(1.52)]
const fn checked_div(l: usize, r: usize) -> Option<usize> {
if r == 0 { None } else { Some(l / r) }
}
#[::rustversion::since(1.52)]
#[inline(always)]
const fn checked_rem(l: usize, r: usize) -> Option<usize> {
l.checked_rem(r)
}
#[::rustversion::before(1.52)]
const fn checked_rem(l: usize, r: usize) -> Option<usize> {
if r == 0 { None } else { Some(l % r) }
}
#[::rustversion::since(1.50)]
#[inline(always)]
const fn checked_pow(l: usize, r: u32) -> Option<usize> {
l.checked_pow(r)
}
#[::rustversion::before(1.50)]
#[::rustversion::attr(since(1.47), const)]
fn checked_pow(l: usize, mut r: u32) -> Option<usize> {
if r == 0 {
return Some(1);
}
let mut base = l;
let mut acc: usize = 1;
loop {
if (r & 1) == 1 {
acc = tri!(opt acc.checked_mul(base));
if r == 1 {
return Some(acc);
}
}
r /= 2;
base = tri!(opt base.checked_mul(base));
}
}
let res = match op {
ArithOp::Add => l.checked_add(r),
ArithOp::Sub => l.checked_sub(r),
ArithOp::Mul => l.checked_mul(r),
ArithOp::Div => checked_div(l, r),
ArithOp::Rem => checked_rem(l, r),
ArithOp::Pow if r > u32::MAX as usize => None,
#[allow(clippy::cast_possible_truncation)]
ArithOp::Pow => checked_pow(l, r as u32)
};
match res {
Some(v) => Ok(v),
None => Err(ArithErr(l, op, r))
}
}
#[must_use]
#[inline]
pub const fn align_up(v: usize, align: usize) -> usize {
let m1 = align - 1;
(v + m1) & !m1
}
#[::rustversion::attr(since(1.47), const)]
pub fn align_up_checked(v: usize, align: usize) -> Result<usize, Error> {
if align == 0 {
return Err(Error::InvalidLayout(v, align, LayoutErr::ZeroAlign));
} else if !align.is_power_of_two() {
return Err(Error::InvalidLayout(v, align, LayoutErr::NonPowerOfTwoAlign));
}
let m1 = align - 1;
Ok(tri!(::ArithmeticError checked_op(v, ArithOp::Add, m1)) & !m1)
}
#[must_use]
#[inline]
pub const unsafe fn dangling_nonnull(align: usize) -> NonNull<u8> {
NonNull::new_unchecked(align as *mut u8)
}
#[cfg_attr(miri, track_caller)]
#[must_use]
#[inline]
pub fn ptr_max_align(ptr: NonNull<u8>) -> usize {
let p = ptr.as_ptr() as usize;
p & p.wrapping_neg()
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
pub fn nonnull_slice_from_parts<T>(p: NonNull<T>, len: usize) -> NonNull<[T]> {
varsized_nonnull_from_parts(p.cast(), len)
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
pub fn slice_ptr_from_parts_mut<T>(p: *mut T, len: usize) -> *mut [T] {
varsized_ptr_from_parts_mut(p.cast(), len)
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
pub fn slice_ptr_from_parts<T>(p: *const T, len: usize) -> *const [T] {
varsized_ptr_from_parts(p.cast(), len)
}
#[::rustversion::attr(since(1.58), const)]
#[must_use]
#[inline]
pub unsafe fn nonnull_slice_len<T>(ptr: NonNull<[T]>) -> usize {
(&*(ptr.as_ptr() as *const [T])).len()
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
pub fn varsized_dangling_nonnull<T: ?Sized + VarSized>() -> NonNull<T> {
varsized_nonnull_from_parts(unsafe { dangling_nonnull(T::ALN) }, 0)
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
pub fn varsized_dangling_ptr<T: ?Sized + VarSized>() -> *mut T {
varsized_ptr_from_parts_mut(unsafe { dangling_nonnull(T::ALN).as_ptr() }, 0)
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn varsized_nonnull_from_parts<T: ?Sized + VarSized>(
p: NonNull<u8>,
meta: usize
) -> NonNull<T> {
unsafe { NonNull::new_unchecked(varsized_ptr_from_parts_mut(p.as_ptr(), meta)) }
}
#[::rustversion::since(1.83)]
#[must_use]
#[inline]
pub const fn varsized_ptr_from_parts_mut<T: ?Sized + VarSized>(p: *mut u8, meta: usize) -> *mut T {
unsafe {
*((&ptr::slice_from_raw_parts_mut::<T::SubType>(p.cast(), meta)
as *const *mut [T::SubType])
.cast::<*mut T>())
}
}
#[::rustversion::before(1.83)]
#[must_use]
#[inline]
#[::rustversion::attr(since(1.61), const)]
pub fn varsized_ptr_from_parts_mut<T: ?Sized + VarSized>(p: *mut u8, meta: usize) -> *mut T {
unsafe { crate::helpers::union_transmute::<(*mut u8, usize), *mut T>((p, meta)) }
}
#[::rustversion::since(1.64)]
#[must_use]
#[inline]
pub const fn varsized_ptr_from_parts<T: ?Sized + VarSized>(p: *const u8, meta: usize) -> *const T {
unsafe {
*((&ptr::slice_from_raw_parts::<T::SubType>(p.cast(), meta)
as *const *const [T::SubType])
.cast::<*const T>())
}
}
#[::rustversion::before(1.64)]
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn varsized_ptr_from_parts<T: ?Sized + VarSized>(p: *const u8, meta: usize) -> *const T {
unsafe { crate::helpers::union_transmute::<(*const u8, usize), *const T>((p, meta)) }
}
pub fn null_q<T>(ptr: *mut T, layout: Layout) -> Result<NonNull<u8>, Error> {
if ptr.is_null() {
Err(Error::AllocFailed(layout, Cause::Unknown))
} else {
Ok(unsafe { NonNull::new_unchecked(ptr.cast()) })
}
}
#[cfg(feature = "os_err_reporting")]
pub fn null_q_dyn<T>(ptr: *mut T, layout: Layout) -> Result<NonNull<u8>, Error> {
if ptr.is_null() {
Err(Error::AllocFailed(
layout,
Cause::OSErr(unsafe {
#[allow(clippy::option_if_let_else)]
match ::std::io::Error::last_os_error().raw_os_error() {
Some(e) => e,
None => ::core::hint::unreachable_unchecked()
}
})
))
} else {
Ok(unsafe { NonNull::new_unchecked(ptr.cast()) })
}
}
#[cfg(not(feature = "os_err_reporting"))]
pub fn null_q_dyn<T>(ptr: *mut T, layout: Layout) -> Result<NonNull<u8>, Error> {
null_q(ptr, layout)
}
pub fn null_q_dyn_zsl_check<T, F: Fn(Layout) -> *mut T>(
layout: Layout,
f: F
) -> Result<NonNull<u8>, Error> {
if layout.is_zero_sized() { Err(Error::ZeroSizedLayout) } else { null_q_dyn(f(layout), layout) }
}
#[::rustversion::since(1.75)]
pub const unsafe fn byte_sub<T: ?Sized>(p: *const T, n: usize) -> *const T {
p.byte_sub(n)
}
#[::rustversion::before(1.75)]
pub unsafe fn byte_sub<T: ?Sized>(p: *const T, n: usize) -> *const T {
let mut p = p;
let addr_ptr = (&mut p as *mut *const T).cast::<usize>();
unsafe {
ptr::write(addr_ptr, *addr_ptr - n);
}
p
}
#[::rustversion::since(1.75)]
pub const unsafe fn byte_add<T: ?Sized>(p: *const T, n: usize) -> *const T {
p.byte_add(n)
}
#[::rustversion::before(1.75)]
pub unsafe fn byte_add<T: ?Sized>(p: *const T, n: usize) -> *const T {
let mut p = p;
let addr_ptr = (&mut p as *mut *const T).cast::<usize>();
unsafe {
ptr::write(addr_ptr, *addr_ptr + n);
}
p
}
#[::rustversion::since(1.49)]
#[::rustversion::attr(since(1.56), const)]
pub unsafe fn union_transmute<Src, Dst>(src: Src) -> Dst {
use ::core::mem::ManuallyDrop;
union Either<Src, Dst> {
src: ManuallyDrop<Src>,
dst: ManuallyDrop<Dst>
}
ManuallyDrop::into_inner(Either { src: ManuallyDrop::new(src) }.dst)
}
#[::rustversion::before(1.49)]
pub unsafe fn union_transmute<Src: Copy, Dst: Copy>(src: Src) -> Dst {
union Either<Src: Copy, Dst: Copy> {
src: Src,
dst: Dst
}
Either { src }.dst
}
#[must_use]
pub const fn is_multiple_of(lhs: usize, rhs: usize) -> bool {
match rhs {
0 => lhs == 0,
_ => lhs % rhs == 0
}
}
pub struct AllocGuard<'a, T: ?Sized, A: BasicAlloc + ?Sized> {
ptr: NonNull<T>,
alloc: &'a A
}
impl<'a, T: ?Sized, A: BasicAlloc + ?Sized> AllocGuard<'a, T, A> {
#[::rustversion::attr(since(1.61), const)]
#[inline]
pub unsafe fn new(ptr: NonNull<T>, alloc: &'a A) -> AllocGuard<'a, T, A> {
AllocGuard { ptr, alloc }
}
#[::rustversion::attr(since(1.83), const)]
#[cfg_attr(miri, track_caller)]
#[inline]
pub fn init(&mut self, elem: T)
where
T: Sized
{
unsafe {
ptr::write(self.ptr.as_ptr(), elem);
}
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn release(self) -> NonNull<T> {
let ptr = self.ptr;
forget(self);
ptr
}
}
impl<T: ?Sized, A: BasicAlloc + ?Sized> Drop for AllocGuard<'_, T, A> {
#[cfg_attr(miri, track_caller)]
fn drop(&mut self) {
let layout = unsafe { self.ptr.layout() };
unsafe {
self.alloc.dealloc(self.ptr.cast::<u8>(), layout);
}
}
}
impl<T: ?Sized, A: BasicAlloc + ?Sized> Deref for AllocGuard<'_, T, A> {
type Target = NonNull<T>;
#[inline]
fn deref(&self) -> &NonNull<T> {
&self.ptr
}
}
pub struct SliceAllocGuard<'a, T, A: BasicAlloc + ?Sized> {
ptr: NonNull<T>,
alloc: &'a A,
pub(crate) init: usize,
full: usize
}
impl<'a, T, A: BasicAlloc + ?Sized> SliceAllocGuard<'a, T, A> {
#[::rustversion::attr(since(1.61), const)]
#[inline]
pub unsafe fn new(ptr: NonNull<T>, alloc: &'a A, full: usize) -> SliceAllocGuard<'a, T, A> {
SliceAllocGuard { ptr, alloc, init: 0, full }
}
#[::rustversion::attr(since(1.61), const)]
#[inline]
pub unsafe fn new_with_init(
ptr: NonNull<T>,
alloc: &'a A,
init: usize,
full: usize
) -> SliceAllocGuard<'a, T, A> {
SliceAllocGuard { ptr, alloc, init, full }
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn release(self) -> NonNull<[T]> {
let ret = self.get_init_part();
forget(self);
ret
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn release_first(self) -> NonNull<T> {
let ret = self.ptr;
forget(self);
ret
}
#[::rustversion::attr(since(1.61), const)]
#[cfg_attr(miri, track_caller)]
#[must_use]
pub fn get_init_part(&self) -> NonNull<[T]> {
nonnull_slice_from_parts(self.ptr, self.init)
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
pub fn get_uninit_part(&self) -> NonNull<[T]> {
let ptr = unsafe { self.ptr.as_ptr().add(self.init) };
nonnull_slice_from_parts(
unsafe { NonNull::new_unchecked(ptr) },
self.full - self.init
)
}
#[::rustversion::attr(since(1.61), const)]
#[cfg_attr(miri, track_caller)]
#[must_use]
pub fn get_full(&self) -> NonNull<[T]> {
nonnull_slice_from_parts(self.ptr, self.full)
}
#[::rustversion::attr(since(1.83), const)]
#[inline]
pub unsafe fn set_init(&mut self, init: usize) {
self.init = init;
}
#[::rustversion::attr(since(1.83), const)]
#[inline]
pub fn init(&mut self, elem: T) -> Result<(), T> {
if self.init == self.full {
return Err(elem);
}
unsafe {
self.init_unchecked(elem);
}
Ok(())
}
#[::rustversion::attr(since(1.83), const)]
#[inline]
pub unsafe fn init_unchecked(&mut self, elem: T) {
ptr::write(self.ptr.as_ptr().add(self.init), elem);
self.init += 1;
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn initialized(&self) -> usize {
self.init
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn full(&self) -> usize {
self.full
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn is_full(&self) -> bool {
self.init == self.full
}
#[::rustversion::attr(since(1.61), const)]
#[must_use]
#[inline]
pub fn is_empty(&self) -> bool {
self.init == 0
}
#[::rustversion::attr(since(1.83), const)]
pub fn copy_from_slice(&mut self, slice: &[T]) -> Result<(), usize>
where
T: Copy
{
let lim = self.full - self.init;
let to_copy = if slice.len() < lim { slice.len() } else { lim };
let uninit_p = unsafe { self.ptr.as_ptr().add(self.init) };
unsafe {
ptr::copy(slice.as_ptr(), uninit_p, to_copy);
}
self.init += to_copy;
let uncopied = slice.len() - to_copy;
if uncopied == 0 { Ok(()) } else { Err(uncopied) }
}
pub fn clone_from_slice(&mut self, slice: &[T]) -> Result<(), usize>
where
T: Clone
{
let lim = self.full - self.init;
let to_clone = if slice.len() < lim { slice.len() } else { lim };
for elem in &slice[..to_clone] {
let ptr = unsafe { self.as_ptr().add(self.init) };
unsafe {
ptr::write(ptr, elem.clone());
}
self.init += 1;
}
self.init += to_clone;
let uncloned = slice.len() - to_clone;
if uncloned == 0 { Ok(()) } else { Err(uncloned) }
}
pub fn extend_init<I: IntoIterator<Item = T>>(&mut self, iter: I) -> Result<(), I::IntoIter> {
let mut iter = iter.into_iter();
loop {
if self.init == self.full && iter.size_hint().0 != 0 {
return Err(iter);
}
match iter.next() {
Some(elem) => {
let uninit_ptr = unsafe { self.ptr.as_ptr().add(self.init) };
unsafe {
ptr::write(uninit_ptr, elem);
self.init += 1;
}
}
None => return Ok(())
}
}
}
}
impl<T, A: BasicAlloc + ?Sized> Drop for SliceAllocGuard<'_, T, A> {
fn drop(&mut self) {
unsafe {
ptr::drop_in_place(slice_ptr_from_parts_mut(self.ptr.as_ptr(), self.init));
}
let layout = unsafe { Layout::from_size_align_unchecked(T::SZ * self.full, T::ALN) };
unsafe {
self.alloc.dealloc(self.ptr.cast(), layout);
}
}
}
impl<T, A: BasicAlloc + ?Sized> Deref for SliceAllocGuard<'_, T, A> {
type Target = NonNull<T>;
#[inline]
fn deref(&self) -> &NonNull<T> {
&self.ptr
}
}