#![deny(unsafe_op_in_unsafe_fn)]
#![warn(missing_docs)]
#![warn(rustdoc::broken_intra_doc_links, rust_2018_idioms)]
#![allow(dead_code)]
#![cfg_attr(not(feature = "std"), no_std)]
extern crate alloc;
mod domain;
mod hazard;
mod pointer;
mod record;
mod sync;
fn asymmetric_light_barrier() {
crate::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
}
enum HeavyBarrierKind {
Normal,
Expedited,
}
fn asymmetric_heavy_barrier(_: HeavyBarrierKind) {
crate::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
}
pub mod raw {
pub use crate::domain::Domain;
pub mod families {
pub use crate::domain::Global;
}
pub use crate::hazard::HazardPointer;
pub use crate::pointer::{Pointer, Reclaim};
}
use std::marker::PhantomData;
use std::ops::Deref;
use std::ops::DerefMut;
use std::ptr::NonNull;
use std::sync::atomic::Ordering;
pub use domain::Domain;
pub use domain::Global;
pub use hazard::{HazardPointer, HazardPointerArray};
#[repr(transparent)]
pub struct AtomicPtr<T, F = domain::Global, P = Box<T>>(
crate::sync::atomic::AtomicPtr<T>,
PhantomData<(F, *mut P)>,
);
impl<T, F, P> From<P> for AtomicPtr<T, F, P>
where
P: raw::Pointer<T>,
{
fn from(p: P) -> Self {
Self(
crate::sync::atomic::AtomicPtr::new(p.into_raw()),
PhantomData,
)
}
}
impl<T, F, P> AtomicPtr<T, F, P> {
pub unsafe fn new(p: *mut T) -> Self {
Self(crate::sync::atomic::AtomicPtr::new(p), PhantomData)
}
pub unsafe fn as_std(&self) -> &crate::sync::atomic::AtomicPtr<T> {
&self.0
}
pub unsafe fn as_std_mut(&mut self) -> &mut crate::sync::atomic::AtomicPtr<T> {
&mut self.0
}
}
#[repr(transparent)]
pub struct Replaced<T, F, P> {
ptr: NonNull<T>,
_family: PhantomData<F>,
_holder: PhantomData<P>,
}
impl<T, F, P> Clone for Replaced<T, F, P> {
fn clone(&self) -> Self {
Self {
ptr: self.ptr,
_family: self._family,
_holder: self._holder,
}
}
}
impl<T, F, P> Copy for Replaced<T, F, P> {}
impl<T, F, P> Deref for Replaced<T, F, P> {
type Target = NonNull<T>;
fn deref(&self) -> &Self::Target {
&self.ptr
}
}
impl<T, F, P> DerefMut for Replaced<T, F, P> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.ptr
}
}
impl<T, F, P> AsRef<NonNull<T>> for Replaced<T, F, P> {
fn as_ref(&self) -> &NonNull<T> {
&self.ptr
}
}
impl<T, F, P> Replaced<T, F, P> {
pub fn into_inner(self) -> NonNull<T> {
self.ptr
}
}
impl<T, P> Replaced<T, raw::families::Global, P>
where
P: raw::Pointer<T>,
{
pub unsafe fn retire(self) -> usize {
unsafe { self.retire_in(Domain::global()) }
}
}
impl<T, F, P> Replaced<T, F, P>
where
P: raw::Pointer<T>,
{
pub unsafe fn retire_in(mut self, domain: &Domain<F>) -> usize {
unsafe { domain.retire_ptr::<T, P>(self.ptr.as_mut()) }
}
}
impl<T, P> AtomicPtr<T, domain::Global, P> {
pub fn safe_load<'hp>(
&'_ self,
hp: &'hp mut HazardPointer<'static, domain::Global>,
) -> Option<&'hp T>
where
T: 'hp,
{
unsafe { self.load(hp) }
}
}
impl<T, F, P> AtomicPtr<T, F, P> {
pub unsafe fn load<'hp, 'd>(&'_ self, hp: &'hp mut HazardPointer<'d, F>) -> Option<&'hp T>
where
T: 'hp,
F: 'static,
{
unsafe { hp.protect(&self.0) }
}
#[cfg(not(loom))]
pub unsafe fn get_mut(&mut self) -> &mut *mut T {
self.0.get_mut()
}
pub fn into_inner(self) -> *mut T {
#[cfg(not(loom))]
let ptr = self.0.into_inner();
#[cfg(loom)]
let ptr = unsafe { self.0.unsync_load() };
ptr
}
}
impl<T, P> AtomicPtr<T, raw::families::Global, P>
where
P: raw::Pointer<T>,
{
pub unsafe fn retire(self) -> usize {
unsafe { self.retire_in(Domain::global()) }
}
}
impl<T, F, P> AtomicPtr<T, F, P>
where
P: raw::Pointer<T>,
{
pub unsafe fn retire_in(self, domain: &Domain<F>) -> usize {
let ptr = self.into_inner();
unsafe { domain.retire_ptr::<T, P>(ptr) }
}
pub fn store(&self, p: P) {
let ptr = p.into_raw();
unsafe { self.store_ptr(ptr) }
}
pub fn swap(&self, p: P) -> Option<Replaced<T, F, P>> {
let ptr = p.into_raw();
unsafe { self.swap_ptr(ptr) }
}
#[allow(clippy::not_unsafe_ptr_arg_deref)]
pub fn compare_exchange(
&self,
current: *mut T,
new: P,
) -> Result<Option<Replaced<T, F, P>>, P> {
let new = new.into_raw();
let r = unsafe { self.compare_exchange_ptr(current, new) };
r.map_err(|ptr| {
unsafe { P::from_raw(ptr) }
})
}
#[allow(clippy::not_unsafe_ptr_arg_deref)]
pub fn compare_exchange_weak(
&self,
current: *mut T,
new: P,
) -> Result<Option<Replaced<T, F, P>>, P> {
let new = new.into_raw();
let r = unsafe { self.compare_exchange_weak_ptr(current, new) };
r.map_err(|ptr| {
unsafe { P::from_raw(ptr) }
})
}
}
impl<T, F, P> AtomicPtr<T, F, P> {
pub fn load_ptr(&self) -> *mut T {
self.0.load(Ordering::Acquire)
}
pub unsafe fn store_ptr(&self, ptr: *mut T) {
self.0.store(ptr, Ordering::Release)
}
pub unsafe fn swap_ptr(&self, ptr: *mut T) -> Option<Replaced<T, F, P>> {
let ptr = self.0.swap(ptr, Ordering::Release);
NonNull::new(ptr).map(|ptr| Replaced {
ptr,
_family: PhantomData::<F>,
_holder: PhantomData::<P>,
})
}
pub unsafe fn compare_exchange_ptr(
&self,
current: *mut T,
new: *mut T,
) -> Result<Option<Replaced<T, F, P>>, *mut T> {
let ptr = self
.0
.compare_exchange(current, new, Ordering::Release, Ordering::Relaxed)?;
Ok(NonNull::new(ptr).map(|ptr| Replaced {
ptr,
_family: PhantomData::<F>,
_holder: PhantomData::<P>,
}))
}
pub unsafe fn compare_exchange_weak_ptr(
&self,
current: *mut T,
new: *mut T,
) -> Result<Option<Replaced<T, F, P>>, *mut T> {
let ptr =
self.0
.compare_exchange_weak(current, new, Ordering::Release, Ordering::Relaxed)?;
Ok(NonNull::new(ptr).map(|ptr| Replaced {
ptr,
_family: PhantomData::<F>,
_holder: PhantomData::<P>,
}))
}
}