#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))]
use alloc::sync::Arc as AllocArc;
use {
crate::{
TypeMeta,
config::ConfigCore,
error::{ReadResult, WriteResult},
io::{Reader, Writer},
len::SeqLen,
schema::{SchemaRead, SchemaWrite, size_of_elem_iter, write_elem_iter},
},
core::{
borrow::Borrow,
marker::PhantomData,
mem::{self, MaybeUninit},
ptr,
},
};
#[cfg(feature = "alloc")]
use {
crate::{
context,
schema::{SchemaReadContext, size_of_elem_slice, write_elem_slice_prealloc_check},
},
alloc::{boxed::Box as AllocBox, collections, rc::Rc as AllocRc, vec},
};
#[cfg(feature = "alloc")]
pub struct Vec<T, Len>(PhantomData<Len>, PhantomData<T>);
#[cfg(feature = "alloc")]
pub struct VecDeque<T, Len>(PhantomData<Len>, PhantomData<T>);
#[cfg(feature = "alloc")]
pub struct Box<T: ?Sized, Len>(PhantomData<T>, PhantomData<Len>);
#[cfg(feature = "alloc")]
pub struct Rc<T: ?Sized, Len>(PhantomData<T>, PhantomData<Len>);
#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))]
pub struct Arc<T: ?Sized, Len>(PhantomData<T>, PhantomData<Len>);
#[macro_export]
macro_rules! pod_wrapper {
($(unsafe struct $name:ident($type:ty);)*) => {$(
struct $name where $type: Copy + 'static;
// SAFETY:
// - By using `pod_wrapper`, user asserts that the type is zero-copy, given the contract of
// pod_wrapper:
// - The type's in‑memory representation is exactly its serialized bytes.
// - It can be safely initialized by memcpy (no validation, no endianness/layout work).
unsafe impl<C: $crate::config::ConfigCore> $crate::config::ZeroCopy<C> for $name {}
unsafe impl<C: $crate::config::ConfigCore> $crate::SchemaWrite<C> for $name {
type Src = $type;
const TYPE_META: $crate::TypeMeta = $crate::TypeMeta::Static {
size: size_of::<$type>(),
zero_copy: true,
};
#[inline]
fn size_of(_: &$type) -> $crate::WriteResult<usize> {
Ok(size_of::<$type>())
}
#[inline]
fn write(mut writer: impl $crate::io::Writer, src: &$type) -> $crate::WriteResult<()> {
unsafe {
Ok(writer.write_t(src)?)
}
}
}
unsafe impl<'de, C: $crate::config::ConfigCore> $crate::SchemaRead<'de, C> for $name {
type Dst = $type;
const TYPE_META: $crate::TypeMeta = $crate::TypeMeta::Static {
size: size_of::<$type>(),
zero_copy: true,
};
fn read(mut reader: impl $crate::io::Reader<'de>, dst: &mut core::mem::MaybeUninit<$type>) -> $crate::ReadResult<()> {
unsafe {
Ok(reader.copy_into_t(dst)?)
}
}
}
)*}
}
pub use pod_wrapper;
#[cfg(feature = "alloc")]
unsafe impl<T, Len, C: ConfigCore> SchemaWrite<C> for Vec<T, Len>
where
Len: SeqLen<C>,
T: SchemaWrite<C>,
T::Src: Sized,
{
type Src = vec::Vec<T::Src>;
#[inline(always)]
fn size_of(src: &Self::Src) -> WriteResult<usize> {
size_of_elem_slice::<T, Len, C>(src)
}
#[inline(always)]
fn write(writer: impl Writer, src: &Self::Src) -> WriteResult<()> {
write_elem_slice_prealloc_check::<T, Len, C>(writer, src)
}
}
#[cfg(feature = "alloc")]
unsafe impl<'de, T, Len, C: ConfigCore> SchemaRead<'de, C> for Vec<T, Len>
where
Len: SeqLen<C>,
T: SchemaRead<'de, C>,
{
type Dst = vec::Vec<T::Dst>;
#[inline]
fn read(mut reader: impl Reader<'de>, dst: &mut MaybeUninit<Self::Dst>) -> ReadResult<()> {
let len = Len::read_prealloc_check::<T::Dst>(reader.by_ref())?;
<vec::Vec<T>>::read_with_context(context::Len(len), reader, dst)?;
Ok(())
}
}
pub(crate) struct SliceDropGuard<T> {
ptr: *mut MaybeUninit<T>,
initialized_len: usize,
}
impl<T> SliceDropGuard<T> {
pub(crate) fn new(ptr: *mut MaybeUninit<T>) -> Self {
Self {
ptr,
initialized_len: 0,
}
}
#[inline(always)]
#[allow(clippy::arithmetic_side_effects)]
pub(crate) fn inc_len(&mut self) {
if mem::needs_drop::<T>() {
self.initialized_len += 1;
}
}
}
impl<T> Drop for SliceDropGuard<T> {
#[cold]
fn drop(&mut self) {
if mem::needs_drop::<T>() {
unsafe {
ptr::drop_in_place(ptr::slice_from_raw_parts_mut(
self.ptr.cast::<T>(),
self.initialized_len,
));
}
}
}
}
#[inline]
#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))]
unsafe fn arc_get_mut_unchecked<T: ?Sized>(arc: &mut AllocArc<T>) -> &mut T {
unsafe { &mut *AllocArc::as_ptr(arc).cast_mut() }
}
#[inline]
#[cfg(feature = "alloc")]
unsafe fn rc_get_mut_unchecked<T: ?Sized>(rc: &mut AllocRc<T>) -> &mut T {
unsafe { &mut *AllocRc::as_ptr(rc).cast_mut() }
}
macro_rules! impl_heap_slice {
($container:ident => $target:ident, |$uninit:ident| $get_slice:expr) => {
#[cfg(feature = "alloc")]
unsafe impl<T, Len, C: ConfigCore> SchemaWrite<C> for $container<[T], Len>
where
Len: SeqLen<C>,
T: SchemaWrite<C>,
T::Src: Sized,
{
type Src = $target<[T::Src]>;
#[inline(always)]
fn size_of(src: &Self::Src) -> WriteResult<usize> {
size_of_elem_slice::<T, Len, C>(src)
}
#[inline(always)]
fn write(writer: impl Writer, src: &Self::Src) -> WriteResult<()> {
write_elem_slice_prealloc_check::<T, Len, C>(writer, src)
}
}
#[cfg(feature = "alloc")]
unsafe impl<'de, T, Len, C: ConfigCore> SchemaRead<'de, C> for $container<[T], Len>
where
Len: SeqLen<C>,
T: SchemaRead<'de, C>,
{
type Dst = $target<[T::Dst]>;
#[inline(always)]
fn read(
mut reader: impl Reader<'de>,
dst: &mut MaybeUninit<Self::Dst>,
) -> ReadResult<()> {
let len = Len::read_prealloc_check::<T::Dst>(reader.by_ref())?;
let mut $uninit = $target::<[T::Dst]>::new_uninit_slice(len);
decode_into_slice_t::<T, C>(reader, $get_slice)?;
let container = unsafe { $uninit.assume_init() };
dst.write(container);
Ok(())
}
}
};
}
impl_heap_slice!(Box => AllocBox, |uninit| &mut *uninit);
impl_heap_slice!(Rc => AllocRc, |uninit| unsafe { rc_get_mut_unchecked(&mut uninit) });
#[cfg(all(feature = "alloc", target_has_atomic = "ptr"))]
impl_heap_slice!(Arc => AllocArc, |uninit| unsafe { arc_get_mut_unchecked(&mut uninit) });
#[cfg(feature = "alloc")]
unsafe impl<T, Len, C: ConfigCore> SchemaWrite<C> for VecDeque<T, Len>
where
Len: SeqLen<C>,
T: SchemaWrite<C>,
T::Src: Sized,
{
type Src = collections::VecDeque<T::Src>;
#[inline(always)]
fn size_of(value: &Self::Src) -> WriteResult<usize> {
size_of_elem_iter::<T, Len, C>(value.iter())
}
#[inline(always)]
fn write(mut writer: impl Writer, src: &Self::Src) -> WriteResult<()> {
if let TypeMeta::Static {
size,
zero_copy: true,
} = T::TYPE_META
{
#[allow(clippy::arithmetic_side_effects)]
let needed = Len::write_bytes_needed_prealloc_check::<T>(src.len())? + src.len() * size;
let mut writer = unsafe { writer.as_trusted_for(needed) }?;
Len::write(writer.by_ref(), src.len())?;
let (front, back) = src.as_slices();
unsafe {
writer.write_slice_t(front)?;
writer.write_slice_t(back)?;
}
writer.finish()?;
return Ok(());
}
write_elem_iter::<T, Len, C>(writer, src.iter())
}
}
#[cfg(feature = "alloc")]
unsafe impl<'de, T, Len, C: ConfigCore> SchemaRead<'de, C> for VecDeque<T, Len>
where
Len: SeqLen<C>,
T: SchemaRead<'de, C>,
{
type Dst = collections::VecDeque<T::Dst>;
#[inline(always)]
fn read(reader: impl Reader<'de>, dst: &mut MaybeUninit<Self::Dst>) -> ReadResult<()> {
let vec = <Vec<T, Len>>::get(reader)?;
dst.write(vec.into());
Ok(())
}
}
#[cfg(feature = "alloc")]
pub struct BinaryHeap<T, Len>(PhantomData<Len>, PhantomData<T>);
#[cfg(feature = "alloc")]
unsafe impl<T, Len, C: ConfigCore> SchemaWrite<C> for BinaryHeap<T, Len>
where
Len: SeqLen<C>,
T: SchemaWrite<C>,
T::Src: Sized,
{
type Src = collections::BinaryHeap<T::Src>;
#[inline(always)]
fn size_of(src: &Self::Src) -> WriteResult<usize> {
size_of_elem_slice::<T, Len, C>(src.as_slice())
}
#[inline(always)]
fn write(writer: impl Writer, src: &Self::Src) -> WriteResult<()> {
write_elem_slice_prealloc_check::<T, Len, C>(writer, src.as_slice())
}
}
#[cfg(feature = "alloc")]
unsafe impl<'de, T, Len, C: ConfigCore> SchemaRead<'de, C> for BinaryHeap<T, Len>
where
Len: SeqLen<C>,
T: SchemaRead<'de, C>,
T::Dst: Ord,
{
type Dst = collections::BinaryHeap<T::Dst>;
#[inline(always)]
fn read(reader: impl Reader<'de>, dst: &mut MaybeUninit<Self::Dst>) -> ReadResult<()> {
let vec = <Vec<T, Len>>::get(reader)?;
dst.write(collections::BinaryHeap::from(vec));
Ok(())
}
}
struct ResultPrealloc<T, E>(Result<T, E>);
impl<A, E, V: FromIterator<A>> FromIterator<Result<A, E>> for ResultPrealloc<V, E> {
fn from_iter<I: IntoIterator<Item = Result<A, E>>>(iter: I) -> ResultPrealloc<V, E> {
struct Iter<I, E> {
inner: I,
error: Option<E>,
}
impl<I: Iterator<Item = Result<T, E>>, T, E> Iterator for Iter<I, E> {
type Item = T;
#[inline]
fn next(&mut self) -> Option<Self::Item> {
self.inner.next()?.map_err(|e| self.error = Some(e)).ok()
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
}
let mut iter = Iter {
inner: iter.into_iter(),
error: None,
};
let result = V::from_iter(&mut iter);
ResultPrealloc(iter.error.map_or(Ok(result), Err))
}
}
trait CollectResultExt<T, E>: Iterator<Item = Result<T, E>> {
#[inline]
fn collect_result_prealloc<B: FromIterator<T>>(self) -> Result<B, E>
where
Self: Sized,
{
self.collect::<ResultPrealloc<B, E>>().0
}
}
impl<T, E, I> CollectResultExt<T, E> for I where I: Iterator<Item = Result<T, E>> {}
pub struct FromIntoIterator<Coll, Len>(PhantomData<(Coll, Len)>);
unsafe impl<Coll, Len, C: ConfigCore> SchemaWrite<C> for FromIntoIterator<Coll, Len>
where
Len: SeqLen<C>,
Coll: IntoIterator,
for<'a> &'a Coll: IntoIterator<Item: SchemaWrite<C>, IntoIter: ExactSizeIterator>,
for<'a> <&'a Coll as IntoIterator>::Item:
Borrow<<<&'a Coll as IntoIterator>::Item as SchemaWrite<C>>::Src>,
{
type Src = Coll;
#[inline]
fn size_of(src: &Coll) -> WriteResult<usize> {
size_of_elem_iter::<<&Coll as IntoIterator>::Item, Len, C>(src.into_iter())
}
#[inline]
fn write(writer: impl Writer, src: &Coll) -> WriteResult<()> {
let iter = src.into_iter();
Len::prealloc_check::<Coll::Item>(iter.len())?;
write_elem_iter::<<&Coll as IntoIterator>::Item, Len, C>(writer, iter)
}
}
unsafe impl<'de, Coll, Len, C: ConfigCore> SchemaRead<'de, C> for FromIntoIterator<Coll, Len>
where
Len: SeqLen<C>,
Coll: IntoIterator<Item: SchemaRead<'de, C>>,
Coll: FromIterator<<Coll::Item as SchemaRead<'de, C>>::Dst>,
{
type Dst = Coll;
#[inline]
fn read(mut reader: impl Reader<'de>, dst: &mut MaybeUninit<Coll>) -> ReadResult<()> {
let len =
Len::read_prealloc_check::<<Coll::Item as SchemaRead<'de, C>>::Dst>(reader.by_ref())?;
let coll = if let TypeMeta::Static { size, .. } = Coll::Item::TYPE_META {
#[allow(clippy::arithmetic_side_effects)]
let mut reader = unsafe { reader.as_trusted_for(size * len) }?;
(0..len)
.map(|_| Coll::Item::get(reader.by_ref()))
.collect_result_prealloc()?
} else {
(0..len)
.map(|_| Coll::Item::get(reader.by_ref()))
.collect_result_prealloc()?
};
dst.write(coll);
Ok(())
}
}
#[inline]
pub fn decode_into_slice_t<'de, T, C>(
mut reader: impl Reader<'de>,
slice: &mut [MaybeUninit<T::Dst>],
) -> ReadResult<()>
where
T: SchemaRead<'de, C>,
C: ConfigCore,
{
let base = slice.as_mut_ptr();
let len = slice.len();
let mut guard = SliceDropGuard::<T::Dst>::new(base);
match T::TYPE_META {
TypeMeta::Static {
zero_copy: true, ..
} => {
unsafe { reader.copy_into_slice_t(slice) }?
}
TypeMeta::Static {
size,
zero_copy: false,
} => {
#[allow(clippy::arithmetic_side_effects)]
let mut reader = unsafe { reader.as_trusted_for(size * len) }?;
for i in 0..len {
let slot = unsafe { &mut *base.add(i) };
T::read(reader.by_ref(), slot)?;
guard.inc_len();
}
}
TypeMeta::Dynamic => {
for i in 0..len {
let slot = unsafe { &mut *base.add(i) };
T::read(reader.by_ref(), slot)?;
guard.inc_len();
}
}
}
mem::forget(guard);
Ok(())
}