use alloc::sync::{Arc, Weak};
use core::sync::atomic::{AtomicU8, Ordering};
#[cfg(feature = "global_counters")]
static GLOBAL_EPOCH_COUNTERS: std::sync::RwLock<Vec<alloc::sync::Weak<EpochCounter>>> =
std::sync::RwLock::new(Vec::new());
#[cfg(feature = "global_counters")]
pub fn register_epoch_counter(epoch_counter: alloc::sync::Weak<EpochCounter>) {
GLOBAL_EPOCH_COUNTERS.write().unwrap().push(epoch_counter)
}
#[cfg(feature = "global_counters")]
pub fn global_counters() -> Vec<::alloc::sync::Weak<EpochCounter>> {
GLOBAL_EPOCH_COUNTERS.read().unwrap().clone()
}
#[cfg(feature = "thread_local_counter")]
thread_local! {
static THREAD_EPOCH_COUNTER: std::cell::OnceCell<std::sync::Arc<EpochCounter>> = const { std::cell::OnceCell::new() };
}
#[cfg(feature = "global_counters")]
pub struct GlobalEpochCounterPool;
#[cfg(feature = "global_counters")]
unsafe impl EpochCounterPool for GlobalEpochCounterPool {
fn wait_for_epochs(&self) {
global_counters.wait_for_epochs()
}
}
#[cfg(feature = "thread_local_counter")]
pub(crate) fn with_thread_local_epoch_counter<T>(fun: impl FnOnce(&EpochCounter) -> T) -> T {
THREAD_EPOCH_COUNTER.with(|epoch_counter| {
let epoch_counter = epoch_counter.get_or_init(|| {
let epoch_counter = Arc::new(EpochCounter::new());
register_epoch_counter(Arc::downgrade(&epoch_counter));
epoch_counter
});
fun(&epoch_counter)
})
}
#[repr(transparent)]
pub struct EpochCounter(core::sync::atomic::AtomicU8);
impl EpochCounter {
#[inline]
pub const fn new() -> Self {
Self(AtomicU8::new(0))
}
#[inline]
pub(crate) fn enter_rcs(&self) {
let old = self.0.fetch_add(1, Ordering::Acquire);
assert!(old % 2 == 0, "Old Epoch counter value should be even!");
}
#[inline]
pub(crate) fn leave_rcs(&self) {
let old = self.0.fetch_add(1, Ordering::Release);
assert!(old % 2 != 0, "Old Epoch counter value should be odd!");
}
pub(crate) fn get_epoch(&self) -> u8 {
self.0.load(Ordering::Acquire)
}
}
pub unsafe trait EpochCounterPool {
fn wait_for_epochs(&self);
}
unsafe impl<F: Fn() -> Vec<Weak<EpochCounter>>> EpochCounterPool for F {
fn wait_for_epochs(&self) {
let epochs = self();
let mut epochs = epochs
.into_iter()
.flat_map(|elem| {
let arc = elem.upgrade()?;
let init_val = arc.get_epoch();
if init_val % 2 == 0 {
return None;
}
Some((init_val, elem))
})
.collect::<Vec<_>>();
while !epochs.is_empty() {
epochs.retain(|elem| {
let Some(arc) = elem.1.upgrade() else {
return false;
};
arc.get_epoch() == elem.0
})
}
}
}
unsafe impl<const N: usize> EpochCounterPool for [Arc<EpochCounter>; N] {
fn wait_for_epochs(&self) {
(|| self.iter().map(Arc::downgrade).collect::<Vec<_>>()).wait_for_epochs()
}
}