use core::{
cell::UnsafeCell,
marker::PhantomData,
mem::MaybeUninit,
ptr,
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
};
use crate::{map::Slot, storage::{FixedStorage, FixedStorageMultiple}};
#[cfg(any(feature = "std", feature = "alloc"))]
use crate::storage::{BoxedSliceStorage, BoxedStorage};
use crate::cache_padded::CachePadded;
#[cfg(feature = "std")]
use rand::Rng;
use crate::map::{self, IMMEDIATE, SimpleLPHashMap, ZERO_OFFSET};
const SUSPENDED_BIT: u64 = 1 << 63;
#[inline]
pub fn gen_without_suspended_bit(generation: u64) -> u64 {
generation & !SUSPENDED_BIT
}
#[inline]
pub fn is_suspended(generation: u64) -> bool {
(generation & SUSPENDED_BIT) != 0
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReserveError {
NoSpace,
FailedGrab,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublishError {
CasFailed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SetGenError {
ReaderNotFound,
ReaderSuspended,
InvalidGeneration,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SuspendError {
ReaderNotFound,
AlreadySuspended,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResumeError {
ReaderNotFound,
GenerationTooOld,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RegisterError {
ReaderCapacityReached,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ReadError;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnregisterError {
ReaderNotFound,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CleanupError {
NothingToClean,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CleanupResult {
pub old_last_valid: u64,
pub new_last_valid: u64,
pub slots_cleaned: u64,
}
pub struct AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>
where
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
write_gen: CachePadded<AtomicU64>,
read_gen: CachePadded<AtomicU64>,
last_valid_gen: CachePadded<AtomicU64>,
free_gen: CachePadded<AtomicU64>,
writers_count: CachePadded<AtomicUsize>,
reader_gens: SimpleLPHashMap<CachePadded<AtomicU64>, SReaderGens>,
buffer_capacity: usize,
buffer: SBuf,
destroy_stages: SAtomicUsizeCounter,
reader_keep_alloc_tickets: SAtomicUsizeCounter,
reader_stage_tickets: SAtomicUsizeCounter,
phantom: PhantomData<T>,
}
unsafe impl<T, SReaderGens, SBuf, SAtomicUsizeCounter> Send for AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>
where
T: Send,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
unsafe impl<T, SReaderGens, SBuf, SAtomicUsizeCounter> Sync for AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>
where
T: Send,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
#[cfg(any(feature = "std", feature = "alloc"))]
impl<T> AreaInner<
T,
BoxedSliceStorage<Slot<CachePadded<AtomicU64>>>,
BoxedSliceStorage<UnsafeCell<MaybeUninit<T>>>,
BoxedStorage<AtomicUsize>
> {
fn new(buffer_capacity: usize, reader_capacity: usize) -> BoxedStorage<Self> {
assert!(buffer_capacity > 0, "buffer_capacity must be > 0");
assert!(
reader_capacity.is_power_of_two(),
"reader_capacity must be a power of two"
);
assert!(reader_capacity > 0, "reader_capacity must be > 0");
let buffer = BoxedSliceStorage::with_capacity_and_init(buffer_capacity, || UnsafeCell::new(MaybeUninit::uninit()));
let destroy_stages = BoxedStorage::new(AtomicUsize::new(2));
let reader_keep_alloc_tickets = BoxedStorage::new(AtomicUsize::new(0));
let reader_stage_tickets = BoxedStorage::new(AtomicUsize::new(0));
let inner = Self {
write_gen: CachePadded::new(AtomicU64::new(0)),
read_gen: CachePadded::new(AtomicU64::new(0)),
last_valid_gen: CachePadded::new(AtomicU64::new(0)),
free_gen: CachePadded::new(AtomicU64::new(0)),
writers_count: CachePadded::new(AtomicUsize::new(0)),
reader_gens: SimpleLPHashMap::with_capacity_and_init(reader_capacity, || {
CachePadded::new(AtomicU64::new(SUSPENDED_BIT))
}),
buffer_capacity,
buffer,
destroy_stages,
reader_keep_alloc_tickets,
reader_stage_tickets,
phantom: PhantomData,
};
BoxedStorage::new(inner)
}
}
impl<T, SReaderGens, SBuf, SAtomicUsizeCounter> AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>
where
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
pub fn init(
area_inner: *mut Self,
reader_gens: SimpleLPHashMap<CachePadded<AtomicU64>, SReaderGens>,
buffer: SBuf,
destroy_stages: SAtomicUsizeCounter,
reader_keep_alloc_tickets: SAtomicUsizeCounter,
reader_stage_tickets: SAtomicUsizeCounter,
) {
unsafe {
(&raw mut (*area_inner).write_gen).write(CachePadded::new(AtomicU64::new(0)));
(&raw mut (*area_inner).read_gen).write(CachePadded::new(AtomicU64::new(0)));
(&raw mut (*area_inner).last_valid_gen).write(CachePadded::new(AtomicU64::new(0)));
(&raw mut (*area_inner).free_gen).write(CachePadded::new(AtomicU64::new(0)));
(&raw mut (*area_inner).writers_count).write(CachePadded::new(AtomicUsize::new(0)));
(&raw mut (*area_inner).reader_gens).write(reader_gens);
(&raw mut (*area_inner).buffer_capacity).write(buffer.capacity());
(&raw mut (*area_inner).buffer).write(buffer);
(&raw mut (*area_inner).destroy_stages).write(destroy_stages);
(&raw mut (*area_inner).reader_keep_alloc_tickets).write(reader_keep_alloc_tickets);
(&raw mut (*area_inner).reader_stage_tickets).write(reader_stage_tickets);
}
}
#[inline]
unsafe fn get_slot_ptr_mut(&self, generation: u64) -> *mut T {
let index = (generation as usize) % self.buffer_capacity;
unsafe { (*self.buffer.slice()[index].get()).as_mut_ptr() }
}
#[inline]
unsafe fn get_slot_ptr_const(&self, generation: u64) -> *const T {
let index = (generation as usize) % self.buffer_capacity;
unsafe { (*self.buffer.slice()[index].get()).as_ptr() }
}
#[inline]
fn load_write_gen(&self) -> u64 {
self.write_gen.load(Ordering::Acquire)
}
#[inline]
fn load_read_gen(&self) -> u64 {
self.read_gen.load(Ordering::Acquire)
}
#[inline]
fn load_last_valid_gen(&self) -> u64 {
self.last_valid_gen.load(Ordering::Acquire)
}
#[inline]
fn load_free_gen(&self) -> u64 {
self.free_gen.load(Ordering::Acquire)
}
#[inline]
fn load_writers_count(&self) -> usize {
self.writers_count.load(Ordering::Acquire)
}
fn try_reserve_slots(&self, n: usize) -> Result<(u64, u64), ReserveError> {
debug_assert!(n != 0, "must reserve at least one slot");
let current_free = self.load_free_gen();
let current_write = self.load_write_gen(); debug_assert!(current_free <= current_write, "free_gen should never exceed write_gen! this should never happen. {} {} {}", current_free, current_write, self.load_read_gen());
let available = self.buffer_capacity as u64 - (self.buffer_capacity as u64).min(current_write - current_free);
if (n as u64) > available {
return Err(ReserveError::NoSpace);
}
let expected_new_write = current_write + n as u64;
match self.write_gen.compare_exchange(
current_write,
expected_new_write,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => Ok((current_write, expected_new_write)),
Err(_) => Err(ReserveError::FailedGrab),
}
}
fn try_reserve_slots_best_effort(&self, n: usize) -> Result<(u64, u64, usize), ReserveError> {
if n == 0 {
return Ok((0, 0, 0));
}
let current_free = self.load_free_gen();
let current_write = self.load_write_gen(); debug_assert!(current_free <= current_write, "free_gen should never exceed write_gen! this should never happen. {} {} {}", current_free, current_write, self.load_read_gen());
let available = self.buffer_capacity as u64 - (self.buffer_capacity as u64).min(current_write - current_free); let actual_n = (n as u64).min(available);
if actual_n == 0 {
return Ok((0, 0, 0));
}
let expected_new_write = current_write + actual_n;
match self.write_gen.compare_exchange(
current_write,
expected_new_write,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => Ok((current_write, expected_new_write, actual_n as usize)),
Err(_) => Err(ReserveError::FailedGrab),
}
}
fn publish_slots(
&self,
start_generation: u64,
end_generation: u64,
) -> Result<(), PublishError> {
if start_generation >= end_generation {
return Ok(());
}
match self.read_gen.compare_exchange(
start_generation,
end_generation,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => Ok(()),
Err(_) => Err(PublishError::CasFailed),
}
}
#[cfg(feature = "std")]
fn register_reader(&self) -> Result<u64, RegisterError> {
let mut rng = rand::rng();
let seed = rng.random::<u64>();
self.register_reader_with_seed(seed)
}
fn register_reader_with_seed(&self, mut seed: u64) -> Result<u64, RegisterError> {
seed = seed & !(1 << 63);
if seed == 0 {
seed = 1;
}
unsafe {
let (ptr, is_new, _, reader_id_raw, index): (*const CachePadded<AtomicU64>, _, _, _, _) = self.reader_gens.get_or_insert_concurrent(
seed,
ZERO_OFFSET,
self.reader_gens.capacity(),
true,
true, );
if ptr.is_null() {
return Err(RegisterError::ReaderCapacityReached);
}
debug_assert!(is_new, "fold mode should always create new entries");
let reader_id = map::key_bits(reader_id_raw);
debug_assert!(reader_id != 0 && (reader_id & (1 << 63)) == 0);
self.reader_gens.finish_init_at(index);
loop {
let current = (**ptr).load(Ordering::Acquire);
debug_assert!(is_suspended(current), "new slot should be suspended");
let entry_generation = gen_without_suspended_bit(current);
match (**ptr).compare_exchange(
current,
entry_generation,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Ok(reader_id),
Err(_) => continue, }
}
}
}
#[inline(always)]
fn get_reader_ptr_boilerplate(
&self,
reader_id: u64,
func_name: &str,
) -> Option<*const CachePadded<AtomicU64>> {
unsafe {
let (ptr, is_new, _, _, _): (*const CachePadded<AtomicU64>, _, _, _, _) = self.reader_gens.get_or_insert_concurrent(
reader_id,
ZERO_OFFSET,
IMMEDIATE,
false,
false,
);
if ptr.is_null() {
return None;
}
debug_assert!(
!is_new,
"{} should not be used to create new readers",
func_name
);
Some(ptr)
}
}
fn get_reader_gen(&self, reader_id: u64) -> Option<u64> {
unsafe {
let ptr = self.get_reader_ptr_boilerplate(reader_id, "get_reader_gen")?;
let generation = (**ptr).load(Ordering::Acquire);
Some(generation)
}
}
fn set_reader_gen(&self, reader_id: u64, new_generation: u64) -> Result<(), SetGenError> {
unsafe {
let ptr = self.get_reader_ptr_boilerplate(reader_id, "set_reader_gen").ok_or(SetGenError::ReaderNotFound)?;
let current = (**ptr).load(Ordering::Acquire);
if is_suspended(current) {
return Err(SetGenError::ReaderSuspended);
}
if new_generation < current {
return Err(SetGenError::InvalidGeneration);
}
(**ptr).store(new_generation, Ordering::Release);
Ok(())
}
}
fn suspend_reader(&self, reader_id: u64) -> Result<(), SuspendError> {
unsafe {
let ptr = self.get_reader_ptr_boilerplate(reader_id, "suspend_reader").ok_or(SuspendError::ReaderNotFound)?;
let current = (**ptr).load(Ordering::Acquire);
if is_suspended(current) {
return Err(SuspendError::AlreadySuspended);
}
let suspended = current | SUSPENDED_BIT;
(**ptr).store(suspended, Ordering::Release);
Ok(())
}
}
fn resume_reader(&self, reader_id: u64) -> Result<bool, ResumeError> {
unsafe {
let ptr = self.get_reader_ptr_boilerplate(reader_id, "resume_reader").ok_or(ResumeError::ReaderNotFound)?;
loop {
let current = (**ptr).load(Ordering::Acquire);
if !is_suspended(current) {
return Ok(false);
}
let expected = current;
let desired = current & !SUSPENDED_BIT;
match (**ptr).compare_exchange(
expected,
desired,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Ok(true),
Err(_) => continue, }
}
}
}
fn resume_reader_at(&self, reader_id: u64, entry_generation: u64) -> Result<bool, ResumeError> {
unsafe {
let ptr = self.get_reader_ptr_boilerplate(reader_id, "resume_reader_at").ok_or(ResumeError::ReaderNotFound)?;
loop {
let current = (**ptr).load(Ordering::Acquire);
if !is_suspended(current) {
return Ok(false);
}
let current_gen_without_suspend = gen_without_suspended_bit(current);
if entry_generation < current_gen_without_suspend {
return Err(ResumeError::GenerationTooOld);
}
let expected = current;
let desired = entry_generation & !SUSPENDED_BIT;
match (**ptr).compare_exchange(
expected,
desired,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Ok(true),
Err(_) => continue, }
}
}
}
fn unregister_reader(&self, reader_id: u64) -> Result<(), UnregisterError> {
match self.suspend_reader(reader_id) {
Ok(_) | Err(SuspendError::AlreadySuspended) => {},
Err(SuspendError::ReaderNotFound) => return Err(UnregisterError::ReaderNotFound),
}
unsafe {
self.reader_gens
.remove_concurrent(reader_id, ZERO_OFFSET, IMMEDIATE);
}
Ok(())
}
fn try_cleanup_old_slots(&self) -> Result<CleanupResult, CleanupError> {
let mut last_valid = self.load_last_valid_gen();
let read_gen = self.load_read_gen();
if last_valid >= read_gen {
return Err(CleanupError::NothingToClean);
}
let mut min_held_gen = read_gen;
unsafe {
let raw_slots = self.reader_gens.raw_slots();
'outer: for slot in raw_slots {
let ptr = (*slot.value.get()).as_ptr();
let mut reader_gen = (**ptr).load(Ordering::Acquire);
'inner: while is_suspended(reader_gen) {
let current_gen_without_suspend = gen_without_suspended_bit(reader_gen);
let forced_update = min_held_gen.max(current_gen_without_suspend);
match (**ptr).compare_exchange(
reader_gen,
forced_update | SUSPENDED_BIT,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => {
continue 'outer;
},
Err(actual) => {
reader_gen = actual;
continue 'inner;
}
}
}
min_held_gen = min_held_gen.min(reader_gen);
}
}
let new_last_valid = min_held_gen;
loop {
if new_last_valid <= last_valid {
return Err(CleanupError::NothingToClean);
}
match self.last_valid_gen.compare_exchange(
last_valid,
new_last_valid,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => {
let slots_cleaned = new_last_valid - last_valid;
if core::mem::needs_drop::<T>() {
for generation in last_valid..new_last_valid {
unsafe {
let ptr = self.get_slot_ptr_mut(generation);
ptr::drop_in_place(ptr);
}
}
}
loop {
match self.free_gen.compare_exchange(
last_valid,
new_last_valid,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => break,
Err(_) => core::hint::spin_loop(), }
}
return Ok(CleanupResult {
old_last_valid: last_valid,
new_last_valid,
slots_cleaned,
});
}
Err(actual) => {
last_valid = actual;
}
}
}
}
fn has_readers(&self) -> bool {
unsafe {
let raw_slots = self.reader_gens.raw_slots();
for slot in raw_slots {
let k = slot.key.load(Ordering::Acquire);
if k != map::EMPTY {
return true;
}
}
}
false
}
}
#[derive(Debug)]
pub struct AreaWriter<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
inner: Area,
phantom: PhantomData<(T, SReaderGens, SBuf, SAtomicUsizeCounter)>,
}
unsafe impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Send for AreaWriter<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
T: Send,
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
unsafe impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Sync for AreaWriter<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
T: Send,
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> AreaWriter<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>> + Copy + Clone,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
#[inline]
pub fn create_writer(&self) -> Self {
unsafe {
let inner = self.inner.as_ref();
inner.writers_count.fetch_add(1, Ordering::AcqRel);
AreaWriter { inner: self.inner, phantom: PhantomData }
}
}
}
pub trait AreaWriterTrait<T> {
fn try_reserve_slots(&self, n: usize) -> Result<(u64, u64), ReserveError>;
fn try_reserve_slots_best_effort(
&self,
n: usize,
) -> Result<(u64, u64, usize), ReserveError>;
fn publish_slots(
&self,
start_generation: u64,
end_generation: u64,
) -> Result<(), PublishError>;
unsafe fn get_slot_ptr(&self, generation: u64) -> *mut T;
}
macro_rules! impl_writer_functionality {
( $($name:ident),* ) => { $( impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> AreaWriterTrait<T> for $name<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
#[inline]
fn try_reserve_slots(&self, n: usize) -> Result<(u64, u64), ReserveError> {
unsafe { self.inner.as_ref().try_reserve_slots(n) }
}
#[inline]
fn try_reserve_slots_best_effort(
&self,
n: usize,
) -> Result<(u64, u64, usize), ReserveError> {
unsafe { self.inner.as_ref().try_reserve_slots_best_effort(n) }
}
#[inline]
fn publish_slots(
&self,
start_generation: u64,
end_generation: u64,
) -> Result<(), PublishError> {
unsafe {
self.inner
.as_ref()
.publish_slots(start_generation, end_generation)
}
}
#[inline]
unsafe fn get_slot_ptr(&self, generation: u64) -> *mut T {
unsafe { self.inner.as_ref().get_slot_ptr_mut(generation) }
}
}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> $name<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>> + Copy + Clone,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
pub fn reserve(&self, n: usize) -> Result<Reservation<'_, Self, T>, ReserveError> {
let (start, end) = self.try_reserve_slots(n)?;
Ok(Reservation {
writer: self,
start_gen: start,
end_gen: end,
published: false,
phantom: PhantomData,
})
}
pub fn reserve_best_effort(&self, n: usize) -> Result<Reservation<'_, Self, T>, ReserveError> {
let (start, end, count) = self.try_reserve_slots_best_effort(n)?;
debug_assert!(end - start == count as u64);
Ok(Reservation {
writer: self,
start_gen: start,
end_gen: end,
published: false,
phantom: PhantomData,
})
}
}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> $name<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
pub fn get_writers_count(&self) -> usize {
unsafe { self.inner.as_ref().load_writers_count() }
}
}
)* };
}
impl_writer_functionality!(AreaWriter, AreaReader);
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> AreaWriter<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
#[inline]
pub fn try_cleanup_old_slots(&self) -> Result<CleanupResult, CleanupError> {
unsafe { self.inner.as_ref().try_cleanup_old_slots() }
}
}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Drop for AreaWriter<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
fn drop(&mut self) {
unsafe {
let inner = self.inner.as_ref();
let prev_count = inner.writers_count.fetch_sub(1, Ordering::AcqRel);
if prev_count == 1 {
let destroy_stages = inner.destroy_stages.as_ref();
let prev_stages = destroy_stages.fetch_sub(1, Ordering::AcqRel);
if prev_stages == 1 {
let _ = inner.try_cleanup_old_slots();
self.inner.as_ref().destroy_stages.deallocate();
self.inner.as_ref().reader_keep_alloc_tickets.deallocate();
self.inner.as_ref().reader_stage_tickets.deallocate();
self.inner.as_ref().buffer.deallocate();
self.inner.deallocate();
}
}
}
}
}
#[derive(Debug)]
pub struct Reservation<'a, W, T>
where
W: AreaWriterTrait<T>
{
writer: &'a W,
start_gen: u64,
end_gen: u64,
published: bool,
phantom: PhantomData<T>,
}
impl<'a, W, T> Reservation<'a, W, T>
where
W: AreaWriterTrait<T>
{
pub fn len(&self) -> usize {
(self.end_gen - self.start_gen) as usize
}
pub fn is_empty(&self) -> bool {
self.start_gen >= self.end_gen
}
pub fn get_mut(&mut self, index: usize) -> Option<&mut MaybeUninit<T>> {
if index >= self.len() {
return None;
}
let generation = self.start_gen + index as u64;
unsafe {
let ptr = self.writer.get_slot_ptr(generation);
let ptr = ptr as *mut MaybeUninit<T>;
Some(&mut *ptr)
}
}
pub unsafe fn publish(mut self) -> Result<(), Self> {
self.published = true;
match self.writer.publish_slots(self.start_gen, self.end_gen) {
Ok(_) => Ok(()),
Err(PublishError::CasFailed) => Err(self),
}
}
pub unsafe fn publish_spin(self) {
let mut reservation = self;
while let Err(returned) = unsafe { reservation.publish() } {
reservation = returned;
core::hint::spin_loop();
}
}
pub fn split_at_mut(mut self, index: usize) -> (Reservation<'a, W, T>, Reservation<'a, W, T>) {
assert!(index <= self.len(), "index out of bounds");
self.published = true;
let split_gen = self.start_gen + index as u64;
let left = Reservation {
writer: self.writer,
start_gen: self.start_gen,
end_gen: split_gen,
published: false,
phantom: PhantomData,
};
let right = Reservation {
writer: self.writer,
start_gen: split_gen,
end_gen: self.end_gen,
published: false,
phantom: PhantomData,
};
(left, right)
}
}
impl<'a, W, T> Drop for Reservation<'a, W, T>
where
W: AreaWriterTrait<T>
{
fn drop(&mut self) {
if !self.published {
panic!(
"Reservation dropped without publishing! Slots {}..{} are leaked/blocked.",
self.start_gen, self.end_gen
);
}
}
}
pub struct AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
inner: Area,
reader_id: u64,
phantom: PhantomData<(T, SReaderGens, SBuf, SAtomicUsizeCounter)>,
}
unsafe impl<T: Send, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Send for AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
unsafe impl<T: Send, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Sync for AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>> + Copy + Clone,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
#[cfg(feature = "std")]
pub fn create_reader(&self) -> Result<Self, RegisterError> {
unsafe {
let inner = self.inner.as_ref();
let reader_id = inner.register_reader()?;
Ok(AreaReader {
inner: self.inner,
reader_id,
phantom: PhantomData,
})
}
}
pub fn create_reader_with_seed(&self, seed: u64) -> Result<Self, RegisterError> {
unsafe {
let inner = self.inner.as_ref();
let reader_id = inner.register_reader_with_seed(seed)?;
Ok(AreaReader {
inner: self.inner,
reader_id,
phantom: PhantomData,
})
}
}
}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
pub fn get_gen(&self) -> u64 {
unsafe {
self.inner
.as_ref()
.get_reader_gen(self.reader_id)
.expect("Reader not found! This should not happen.")
}
}
pub fn load_read_gen(&self) -> u64 {
unsafe { self.inner.as_ref().load_read_gen() }
}
fn advance(&mut self, n: u64) -> Result<(), SetGenError> {
let current = self.get_gen();
let new_generation = gen_without_suspended_bit(current) + n;
unsafe {
self.inner
.as_ref()
.set_reader_gen(self.reader_id, new_generation)
}
}
unsafe fn get_slot_ptr(&self, generation: u64) -> *const T {
unsafe { self.inner.as_ref().get_slot_ptr_const(generation) }
}
pub fn suspend(&mut self) -> Result<(), SuspendError> {
unsafe { self.inner.as_ref().suspend_reader(self.reader_id) }
}
pub fn resume(&mut self) -> Result<bool, ResumeError> {
unsafe { self.inner.as_ref().resume_reader(self.reader_id) }
}
pub fn resume_at(&mut self, entry_generation: u64) -> Result<bool, ResumeError> {
unsafe {
self.inner
.as_ref()
.resume_reader_at(self.reader_id, entry_generation)
}
}
#[inline]
pub fn try_cleanup_old_slots(&self) -> Result<CleanupResult, CleanupError> {
unsafe { self.inner.as_ref().try_cleanup_old_slots() }
}
}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Drop for AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
fn drop(&mut self) {
unsafe {
let inner = self.inner.as_ref();
let keep_alloc = inner.reader_keep_alloc_tickets.as_ref();
keep_alloc.fetch_add(1, Ordering::AcqRel);
let _ = inner.unregister_reader(self.reader_id);
let has_readers = inner.has_readers();
let attempting_free = !has_readers;
if attempting_free {
let stage_tickets = inner.reader_stage_tickets.as_ref();
stage_tickets.fetch_add(1, Ordering::AcqRel);
}
keep_alloc.fetch_sub(1, Ordering::AcqRel);
if !attempting_free {
return;
}
while keep_alloc.load(Ordering::Acquire) != 0 {
core::hint::spin_loop();
}
let stage_tickets = inner.reader_stage_tickets.as_ref();
let prev_stage = stage_tickets.fetch_sub(1, Ordering::AcqRel);
if prev_stage == 1 {
let destroy_stages = inner.destroy_stages.as_ref();
let prev_stages = destroy_stages.fetch_sub(1, Ordering::AcqRel);
if prev_stages == 1 {
let _ = inner.try_cleanup_old_slots();
self.inner.as_ref().destroy_stages.deallocate();
self.inner.as_ref().reader_keep_alloc_tickets.deallocate();
self.inner.as_ref().reader_stage_tickets.deallocate();
self.inner.as_ref().buffer.deallocate();
self.inner.deallocate(); }
}
}
}
}
pub struct ReadSlice<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
reader: &'a mut AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>,
start_gen: u64,
end_gen: u64,
armed: bool,
}
impl<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> ReadSlice<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
pub unsafe fn new_disarmed(
reader: &'a mut AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>,
start_gen: u64,
end_gen: u64,
) -> Self {
Self {
reader,
start_gen,
end_gen,
armed: false,
}
}
pub unsafe fn disarm(&mut self) {
self.armed = false;
}
pub unsafe fn into_raw_parts(self) -> (&'a mut AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>, u64, u64, bool) {
let manually_dropped_s = core::mem::ManuallyDrop::new(self);
let reader;
let start_gen;
let end_gen;
let armed;
unsafe {
reader = core::ptr::read(&raw const manually_dropped_s.reader);
start_gen = core::ptr::read(&raw const manually_dropped_s.start_gen);
end_gen = core::ptr::read(&raw const manually_dropped_s.end_gen);
armed = core::ptr::read(&raw const manually_dropped_s.armed);
}
(reader, start_gen, end_gen, armed)
}
pub fn len(&self) -> usize {
(self.end_gen - self.start_gen) as usize
}
pub fn is_empty(&self) -> bool {
self.start_gen >= self.end_gen
}
pub fn get(&self, index: usize) -> Option<&T> {
if index >= self.len() {
return None;
}
let generation = self.start_gen + index as u64;
unsafe {
let ptr = self.reader.get_slot_ptr(generation);
Some(&*ptr)
}
}
pub fn iter(&self) -> ReadSliceIter<'_, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> {
ReadSliceIter {
slice: self,
current: 0,
}
}
pub fn as_slices(&self) -> (&[T], &[T]) {
if self.is_empty() {
return (&[], &[]);
}
let cap = unsafe { self.reader.inner.as_ref().buffer_capacity };
let start_idx = (self.start_gen as usize) % cap;
let len = (self.end_gen - self.start_gen) as usize;
unsafe {
let buffer_ptr = self.reader.inner.as_ref().buffer.as_ptr().as_ptr() as *const T;
if start_idx + len <= cap {
let slice = core::slice::from_raw_parts(buffer_ptr.add(start_idx), len);
(slice, &[])
} else {
let first_len = cap - start_idx;
let second_len = len - first_len;
let first = core::slice::from_raw_parts(buffer_ptr.add(start_idx), first_len);
let second = core::slice::from_raw_parts(buffer_ptr, second_len);
(first, second)
}
}
}
#[inline]
pub fn try_cleanup_old_slots(&self) -> Result<CleanupResult, CleanupError> {
unsafe { self.reader.inner.as_ref().try_cleanup_old_slots() }
}
}
impl<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Drop for ReadSlice<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
fn drop(&mut self) {
if !self.armed {
return;
}
let current_gen = gen_without_suspended_bit(self.reader.get_gen());
if current_gen != self.start_gen {
panic!(
"ReadSlice dropped with reader generation mismatch: expected {}, found {}. \
Reader was manually advanced while ReadSlice was alive.",
self.start_gen, current_gen
);
}
let n = self.end_gen - self.start_gen;
if n > 0 {
let _ = self.reader.advance(n);
}
}
}
impl<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> AreaReader<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
pub fn read(&mut self) -> ReadSlice<'_, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> {
let start_generation = gen_without_suspended_bit(self.get_gen());
let end_generation = self.load_read_gen();
ReadSlice {
reader: self,
start_gen: start_generation,
end_gen: end_generation,
armed: true,
}
}
pub fn read_with_check(&mut self) -> Result<ReadSlice<'_, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>, ReadError> {
let (returned_self, start_gen, _, _) = {
let result = self.read();
if result.len() != 0 {
return Ok(result);
}
unsafe {
result.into_raw_parts()
}
};
if returned_self.get_writers_count() == 0 {
let result = returned_self.read();
if result.len() != 0 {
return Ok(result);
}
return Err(ReadError);
}
unsafe {
return Ok(ReadSlice::new_disarmed(returned_self, start_gen, start_gen));
}
}
}
pub struct ReadSliceIter<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
slice: &'a ReadSlice<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>,
current: usize,
}
impl<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> Iterator for ReadSliceIter<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
type Item = &'a T;
fn next(&mut self) -> Option<Self::Item> {
let item = self.slice.get(self.current)?;
self.current += 1;
Some(item)
}
fn size_hint(&self) -> (usize, Option<usize>) {
let remaining = self.slice.len() - self.current;
(remaining, Some(remaining))
}
}
impl<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter> ExactSizeIterator for ReadSliceIter<'a, T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>>,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{}
#[cfg(any(feature = "std", feature = "alloc"))]
pub fn area<T>(buffer_capacity: usize, reader_capacity: usize) -> (
AreaWriter<
T,
BoxedStorage<AreaInner<
T,
BoxedSliceStorage<Slot<CachePadded<AtomicU64>>>,
BoxedSliceStorage<UnsafeCell<MaybeUninit<T>>>,
BoxedStorage<AtomicUsize>,
>>,
BoxedSliceStorage<Slot<CachePadded<AtomicU64>>>,
BoxedSliceStorage<UnsafeCell<MaybeUninit<T>>>,
BoxedStorage<AtomicUsize>
>,
AreaReader<
T,
BoxedStorage<AreaInner<
T,
BoxedSliceStorage<Slot<CachePadded<AtomicU64>>>,
BoxedSliceStorage<UnsafeCell<MaybeUninit<T>>>,
BoxedStorage<AtomicUsize>,
>>,
BoxedSliceStorage<Slot<CachePadded<AtomicU64>>>,
BoxedSliceStorage<UnsafeCell<MaybeUninit<T>>>,
BoxedStorage<AtomicUsize>
>
) {
let inner = AreaInner::new(buffer_capacity, reader_capacity);
unsafe {
let inner_ref = inner.as_ref();
inner_ref.writers_count.store(1, Ordering::Release);
let reader_id = match inner_ref.register_reader_with_seed(1) {
Ok(id) => id,
Err(RegisterError::ReaderCapacityReached) => {
panic!("Initial reader registration failed: capacity reached")
}
};
let writer = AreaWriter { inner, phantom: PhantomData };
let reader = AreaReader { inner, reader_id, phantom: PhantomData };
(writer, reader)
}
}
pub fn finish_init<T, Area, SReaderGens, SBuf, SAtomicUsizeCounter>(inner: Area) -> (
AreaWriter<
T,
Area,
SReaderGens,
SBuf,
SAtomicUsizeCounter,
>,
AreaReader<
T,
Area,
SReaderGens,
SBuf,
SAtomicUsizeCounter,
>
)
where
Area: FixedStorage<AreaInner<T, SReaderGens, SBuf, SAtomicUsizeCounter>> + Clone,
SReaderGens: FixedStorage<Slot<CachePadded<AtomicU64>>> + FixedStorageMultiple<Slot<CachePadded<AtomicU64>>>,
SBuf: FixedStorage<UnsafeCell<MaybeUninit<T>>> + FixedStorageMultiple<UnsafeCell<MaybeUninit<T>>>,
SAtomicUsizeCounter: FixedStorage<AtomicUsize>,
{
unsafe {
let inner_ref = inner.as_ref();
inner_ref.writers_count.store(1, Ordering::Release);
let reader_id = match inner_ref.register_reader_with_seed(1) {
Ok(id) => id,
Err(RegisterError::ReaderCapacityReached) => {
panic!("Initial reader registration failed: capacity reached")
}
};
let writer = AreaWriter { inner: inner.clone(), phantom: PhantomData };
let reader = AreaReader { inner, reader_id, phantom: PhantomData };
(writer, reader)
}
}
#[cfg(test)]
#[cfg(any(feature = "std", feature = "alloc"))]
mod tests {
use super::*;
#[cfg(all(feature = "alloc", not(feature = "std")))]
extern crate alloc;
#[cfg(feature = "std")]
extern crate std as alloc;
#[cfg(any(feature = "std", feature = "alloc"))]
use alloc::vec::Vec;
#[test]
fn test_basic_writer_reserve_and_publish() {
let (writer, reader) = area::<u64>(16, 8);
assert_eq!(reader.get_gen(), 0);
let (start, end) = writer.try_reserve_slots(4).expect("Failed to reserve");
assert_eq!(start, 0);
assert_eq!(end, 4);
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation * 10);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
unsafe {
let read_gen = writer.inner.as_ref().load_read_gen();
assert_eq!(read_gen, 4);
}
}
#[test]
fn test_reservation_write_and_publish() {
let (writer, _reader) = area::<u64>(16, 8);
let mut reservation = writer.reserve(4).unwrap();
for i in 0..4 {
reservation.get_mut(i).unwrap().write(100 + i as u64);
}
unsafe { reservation.publish() }.unwrap_or_else(|_| panic!("Failed to publish reservation"));
}
#[test]
fn test_reservation_split() {
let (writer, _reader) = area::<u64>(16, 8);
let reservation = writer.reserve(4).unwrap();
assert_eq!(reservation.len(), 4);
let (mut left, mut right) = reservation.split_at_mut(2);
assert_eq!(left.len(), 2);
assert_eq!(right.len(), 2);
left.get_mut(0).unwrap().write(100);
left.get_mut(1).unwrap().write(101);
right.get_mut(0).unwrap().write(102);
right.get_mut(1).unwrap().write(103);
match unsafe { right.publish() } {
Ok(_) => panic!("Wrong order works; there's a bug"),
Err(right) => {
assert!(unsafe { left.publish() }.is_ok());
assert!(unsafe { right.publish() }.is_ok());
},
}
}
#[test]
fn test_reservation_raii() {
let (writer, reader) = area::<u64>(16, 8);
let mut reservation = writer.reserve(4).expect("Failed to reserve");
assert_eq!(reservation.len(), 4);
for i in 0..4 {
reservation.get_mut(i).unwrap().write((i as u64) * 100);
}
unsafe { reservation.publish_spin() };
let slice = reader.load_read_gen();
assert_eq!(slice, 4);
}
#[test]
fn test_reservation_inplace_init() {
#[derive(Debug)]
struct ComplexStruct {
a: u64,
b: Vec<u8>,
}
static DROP_COUNT: AtomicUsize = AtomicUsize::new(0);
impl Drop for ComplexStruct {
fn drop(&mut self) {
DROP_COUNT.fetch_add(1, Ordering::SeqCst);
}
}
let (writer, mut reader) = area::<ComplexStruct>(16, 8);
let mut reservation = writer.reserve(1).unwrap();
let slot = reservation.get_mut(0).unwrap();
let ptr = slot.as_mut_ptr();
unsafe {
(&raw mut (*ptr).a).write(42);
(&raw mut (*ptr).b).write(alloc::vec![1, 2, 3]);
}
unsafe { reservation.publish() }.unwrap_or_else(|_| panic!("Failed to publish reservation"));
let slice = reader.read_with_check().expect("There should be writers!");
let (s1, _) = slice.as_slices();
assert_eq!(s1.len(), 1);
assert_eq!(s1[0].a, 42);
assert_eq!(s1[0].b, alloc::vec![1, 2, 3]);
}
#[test]
#[should_panic(expected = "Reservation dropped without publishing")]
fn test_reservation_panic_on_drop() {
let (writer, _reader) = area::<u64>(16, 8);
let _reservation = writer.reserve(4).expect("Failed to reserve");
}
#[test]
fn test_reader_registration_and_reading() {
let (writer, reader) = area::<u64>(16, 8);
let (start, end) = writer.try_reserve_slots(3).expect("Failed to reserve");
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation + 100);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
let reader_gen = reader.get_gen();
assert_eq!(reader_gen, 0);
let read_gen = reader.load_read_gen();
assert_eq!(read_gen, 3);
unsafe {
for generation in reader_gen..read_gen {
let ptr = reader.get_slot_ptr(generation);
let value = ptr.read();
assert_eq!(value, generation + 100);
}
}
}
#[test]
fn test_register_capacity_reached() {
let (_writer, reader1) = area::<u64>(16, 2);
let _reader2 = reader1.create_reader_with_seed(100).expect("Should succeed");
let result = reader1.create_reader_with_seed(101);
assert_eq!(result.err(), Some(RegisterError::ReaderCapacityReached));
}
#[test]
fn test_create_writer() {
let (writer1, _reader) = area::<u64>(16, 8);
let writer2 = writer1.create_writer();
let (start1, end1) = writer1.try_reserve_slots(2).expect("Failed to reserve");
assert_eq!(start1, 0);
assert_eq!(end1, 2);
let (start2, end2) = writer2.try_reserve_slots(2).expect("Failed to reserve");
assert_eq!(start2, 2);
assert_eq!(end2, 4);
}
#[cfg(feature = "std")]
#[test]
fn test_create_reader() {
let (writer, reader1) = area::<u64>(16, 8);
let reader2 = reader1.create_reader().expect("Failed to create reader");
assert_ne!(reader1.reader_id, reader2.reader_id);
let (start, end) = writer.try_reserve_slots(3).expect("Failed to reserve");
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation * 2);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
let read_gen = reader1.load_read_gen();
assert_eq!(read_gen, 3);
assert_eq!(reader2.load_read_gen(), 3);
}
#[test]
fn test_reserve_overflow() {
let (writer, _reader) = area::<u64>(4, 8);
let (start1, end1) = writer.try_reserve_slots(4).expect("Failed to reserve");
assert_eq!(start1, 0);
assert_eq!(end1, 4);
let result = writer.try_reserve_slots(1);
assert_eq!(result, Err(ReserveError::NoSpace));
}
#[test]
fn test_best_effort_reserve() {
let (writer, _reader) = area::<u64>(4, 8);
let (start1, end1) = writer.try_reserve_slots(2).expect("Failed to reserve");
assert_eq!(start1, 0);
assert_eq!(end1, 2);
let (start2, end2, actual) = writer
.try_reserve_slots_best_effort(10)
.expect("Failed to reserve");
assert_eq!(start2, 2);
assert_eq!(end2, 4);
assert_eq!(actual, 2);
let (start3, end3, actual2) = writer
.try_reserve_slots_best_effort(10)
.expect("Failed to reserve");
assert_eq!(start3, 0);
assert_eq!(end3, 0);
assert_eq!(actual2, 0);
assert_eq!(writer.try_reserve_slots(2), Err(ReserveError::NoSpace));
}
#[test]
fn test_suspend_and_resume() {
let (_writer, mut reader) = area::<u64>(16, 8);
assert!(reader.suspend().is_ok());
let generation = reader.get_gen();
assert!(is_suspended(generation));
reader.resume().expect("Failed to resume");
let generation = reader.get_gen();
assert!(!is_suspended(generation));
}
#[test]
fn test_resume_at() {
let (_writer, mut reader) = area::<u64>(16, 8);
assert!(reader.suspend().is_ok());
reader.resume_at(0).expect("Failed to resume");
assert_eq!(reader.get_gen(), 0);
assert!(reader.suspend().is_ok());
reader.resume_at(0).expect("Failed to resume");
}
#[test]
fn test_cleanup_simple() {
let (writer, mut reader) = area::<u64>(16, 8);
let (start, end) = writer.try_reserve_slots(5).expect("Failed to reserve");
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
reader.advance(5).expect("Failed to advance");
let result = writer.try_cleanup_old_slots();
assert!(result.is_ok());
let cleanup_result = result.unwrap();
assert_eq!(cleanup_result.old_last_valid, 0);
assert_eq!(cleanup_result.new_last_valid, 5);
assert_eq!(cleanup_result.slots_cleaned, 5);
}
#[test]
fn test_unregister_unblocks_writer() {
let (writer, reader) = area::<u64>(4, 8);
let (start, end) = writer.try_reserve_slots(4).expect("Failed to reserve");
writer.publish_slots(start, end).expect("Failed to publish");
assert_eq!(writer.try_reserve_slots(1), Err(ReserveError::NoSpace));
drop(reader);
let cleanup_result = writer
.try_cleanup_old_slots()
.expect("Cleanup should succeed");
assert_eq!(cleanup_result.new_last_valid, 4);
let (start, end) = writer.try_reserve_slots(4).expect("Failed to reserve");
assert_eq!(start, 4);
assert_eq!(end, 8);
}
#[test]
fn test_writer_drop_decrements_count() {
let (writer1, _reader) = area::<u64>(16, 8);
unsafe {
let count_before = writer1.inner.as_ref().writers_count.load(Ordering::Acquire);
assert_eq!(count_before, 1);
}
{
let writer2 = writer1.create_writer();
unsafe {
let count_with_two = writer1.inner.as_ref().writers_count.load(Ordering::Acquire);
assert_eq!(count_with_two, 2);
}
drop(writer2);
}
unsafe {
let count_after = writer1.inner.as_ref().writers_count.load(Ordering::Acquire);
assert_eq!(count_after, 1);
}
}
#[test]
fn test_suspend_idempotent() {
let (_writer, mut reader) = area::<u64>(16, 8);
assert!(reader.suspend().is_ok());
assert!(is_suspended(reader.get_gen()));
match reader.suspend() {
Ok(_) => panic!("Suspending again should error"),
Err(SuspendError::ReaderNotFound) => panic!("Reader unregistered!"),
Err(SuspendError::AlreadySuspended) => {},
}
assert!(is_suspended(reader.get_gen()));
}
#[test]
fn test_advance_while_suspended_fails() {
let (_writer, mut reader) = area::<u64>(16, 8);
assert!(reader.suspend().is_ok());
let result = reader.advance(1);
assert_eq!(result, Err(SetGenError::ReaderSuspended));
}
#[test]
fn test_publish_cas_failure() {
let (writer1, _reader) = area::<u64>(16, 8);
let writer2 = writer1.create_writer();
let (start1, end1) = writer1.try_reserve_slots(2).expect("Failed to reserve");
unsafe {
for generation in start1..end1 {
let ptr = writer1.get_slot_ptr(generation);
ptr.write(generation);
}
}
let (start2, end2) = writer2.try_reserve_slots(2).expect("Failed to reserve");
unsafe {
for generation in start2..end2 {
let ptr = writer2.get_slot_ptr(generation);
ptr.write(generation);
}
}
let result = writer2.publish_slots(start2, end2);
assert_eq!(result, Err(PublishError::CasFailed));
writer1
.publish_slots(start1, end1)
.expect("Failed to publish");
writer2
.publish_slots(start2, end2)
.expect("Failed to publish");
let result = writer1.publish_slots(start1, end1);
assert_eq!(result, Err(PublishError::CasFailed));
}
#[test]
fn test_drop_cleanup() {
{
let (writer, reader) = area::<u64>(16, 8);
let _writer2 = writer.create_writer();
let _reader2 = reader.create_reader_with_seed(100).expect("Failed to create reader");
let (start, end) = writer.try_reserve_slots(2).expect("Failed to reserve");
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
}
}
#[test]
fn test_read_slice() {
let (writer, mut reader) = area::<u64>(16, 8);
let (start, end) = writer.try_reserve_slots(5).expect("Failed to reserve");
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation * 100);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
{
let slice = reader.read_with_check().expect("There should be writers!");
assert_eq!(slice.len(), 5);
assert!(!slice.is_empty());
assert_eq!(*slice.get(0).unwrap(), 0);
assert_eq!(*slice.get(1).unwrap(), 100);
assert_eq!(*slice.get(4).unwrap(), 400);
assert!(slice.get(5).is_none());
let values: Vec<u64> = slice.iter().copied().collect();
assert_eq!(values, alloc::vec![0, 100, 200, 300, 400]);
let (s1, s2) = slice.as_slices();
assert_eq!(s1.len() + s2.len(), 5);
assert_eq!(s1.len(), 5);
assert_eq!(s2.len(), 0);
assert_eq!(s1, &[0, 100, 200, 300, 400]);
}
let reader_gen = reader.get_gen();
assert_eq!(reader_gen, 5);
let (start, end) = writer.try_reserve_slots(2).expect("Failed to reserve");
unsafe {
for generation in start..end {
let ptr = writer.get_slot_ptr(generation);
ptr.write(generation * 10);
}
}
writer.publish_slots(start, end).expect("Failed to publish");
{
let slice = reader.read_with_check().expect("There should be writers!");
assert_eq!(slice.len(), 2);
assert_eq!(*slice.get(0).unwrap(), 50);
assert_eq!(*slice.get(1).unwrap(), 60);
}
assert_eq!(reader.get_gen(), 7);
}
#[test]
fn test_register_reader_respects_cleanup() {
let (writer, mut reader1) = area::<u64>(16, 8);
let (start, end) = writer.try_reserve_slots(10).unwrap();
unsafe {
for i in start..end {
writer.get_slot_ptr(i).write(i);
}
}
writer.publish_slots(start, end).unwrap();
reader1.advance(10).unwrap();
let result = writer.try_cleanup_old_slots().unwrap();
assert_eq!(result.new_last_valid, 10);
let reader2 = reader1.create_reader_with_seed(100).expect("Failed to create reader");
assert_eq!(reader2.get_gen(), 10);
}
#[test]
#[should_panic(expected = "reader_capacity must be a power of two")]
fn test_area_creation_panics_on_invalid_capacity() {
let (_writer, _reader) = area::<u64>(16, 0);
}
}