use std::{
cell::{Cell, RefCell},
ptr,
sync::{
Weak,
atomic::{Ordering, fence},
},
};
use crate::EpochEntry;
const MAX_LOCAL_ENTRIES: usize = 4;
const MAX_OVERFLOW_STATES: usize = 16;
#[derive(Clone)]
struct LocalEntryState {
instance_id: u64,
active_entry: usize,
cached_slot: usize,
entries: Option<Weak<[EpochEntry]>>,
}
struct LocalEpochEntries {
count: usize,
inline: [LocalEntryState; MAX_LOCAL_ENTRIES],
overflow: Vec<LocalEntryState>,
}
impl Drop for LocalEpochEntries {
fn drop(&mut self) {
let mut need_fence = false;
for state in self.inline[..self.count]
.iter_mut()
.chain(self.overflow.iter_mut())
{
if state.active_entry != 0
&& let Some(entries) = state.entries.as_ref().and_then(Weak::upgrade)
&& state.active_entry <= entries.len()
{
entries[state.active_entry - 1].reset();
need_fence = true;
}
state.active_entry = 0;
}
if need_fence {
fence(Ordering::SeqCst);
}
}
}
impl LocalEpochEntries {
const fn new() -> Self {
Self {
count: 0,
inline: [const {
LocalEntryState {
instance_id: 0,
active_entry: 0,
cached_slot: 0,
entries: None,
}
}; MAX_LOCAL_ENTRIES],
overflow: Vec::new(),
}
}
#[inline]
fn find(&self, instance_id: u64) -> Option<(usize, usize)> {
self.inline[..self.count]
.iter()
.chain(self.overflow.iter())
.find(|s| s.instance_id == instance_id)
.map(|s| (s.active_entry, s.cached_slot))
}
#[inline]
fn find_mut(&mut self, instance_id: u64) -> Option<&mut LocalEntryState> {
self.inline[..self.count]
.iter_mut()
.chain(self.overflow.iter_mut())
.find(|s| s.instance_id == instance_id)
}
fn set_active<F>(&mut self, instance_id: u64, entry: usize, get_weak: F)
where
F: FnOnce() -> Weak<[EpochEntry]>,
{
if let Some(item) = self.find_mut(instance_id) {
item.active_entry = entry;
if entry != 0 {
item.cached_slot = entry;
if item.entries.as_ref().is_none_or(|e| e.strong_count() == 0) {
item.entries = Some(get_weak());
}
}
return;
}
let new_state = LocalEntryState {
instance_id,
active_entry: entry,
cached_slot: entry,
entries: (entry != 0).then(get_weak),
};
if self.count < MAX_LOCAL_ENTRIES {
self.inline[self.count] = new_state;
self.count += 1;
} else {
if self.overflow.len() >= MAX_OVERFLOW_STATES {
self.overflow.retain(|s| {
s.active_entry != 0 || s.entries.as_ref().is_some_and(|w| w.strong_count() > 0)
});
}
self.overflow.push(new_state);
}
}
}
thread_local! {
static THREAD_LOCAL_ENTRIES: RefCell<LocalEpochEntries> = const { RefCell::new(LocalEpochEntries::new()) };
pub(crate) static FAST_ENTRY: Cell<FastEntry> = const { Cell::new(FastEntry::EMPTY) };
pub(crate) static FAST_PARTICIPANT: Cell<FastEntry> = const { Cell::new(FastEntry::EMPTY) };
}
#[derive(Clone, Copy)]
pub(crate) struct FastEntry {
pub(crate) instance_id: u64,
pub(crate) slot: usize,
pub(crate) ptr: *const EpochEntry,
}
impl FastEntry {
pub(crate) const EMPTY: Self = Self {
instance_id: 0,
slot: 0,
ptr: ptr::null(),
};
}
#[inline]
pub(crate) fn note_participant_slot(instance_id: u64, idx: usize, entry: &EpochEntry) {
FAST_PARTICIPANT.set(FastEntry {
instance_id,
slot: idx + 1,
ptr: ptr::from_ref(entry),
});
}
#[inline]
pub(crate) fn get_thread_entry(instance_id: u64) -> usize {
THREAD_LOCAL_ENTRIES.with(|cell| {
cell
.borrow()
.find(instance_id)
.map_or(0, |(active, _)| active)
})
}
#[inline]
pub(crate) fn cached_slot(instance_id: u64) -> usize {
THREAD_LOCAL_ENTRIES.with(|cell| {
cell
.borrow()
.find(instance_id)
.map_or(0, |(_, cached)| cached)
})
}
#[inline]
pub(crate) fn set_thread_entry<F>(instance_id: u64, entry: usize, get_weak: F)
where
F: FnOnce() -> Weak<[EpochEntry]>,
{
let resolved = THREAD_LOCAL_ENTRIES.with(|cell| {
let mut entries = cell.borrow_mut();
entries.set_active(instance_id, entry, get_weak);
if entry != 0 {
entries
.find_mut(instance_id)
.and_then(|s| s.entries.as_ref())
.and_then(Weak::upgrade)
.filter(|e| entry <= e.len())
.map(|e| ptr::from_ref(&e[entry - 1]))
} else {
None
}
});
match resolved {
Some(p) => FAST_ENTRY.set(FastEntry {
instance_id,
slot: entry,
ptr: p,
}),
None => FAST_ENTRY.set(FastEntry::EMPTY),
}
}
#[inline]
pub(crate) fn clear_thread_entry(instance_id: u64) {
THREAD_LOCAL_ENTRIES.with(|cell| {
let mut entries = cell.borrow_mut();
if let Some(item) = entries.find_mut(instance_id) {
item.active_entry = 0;
}
});
if FAST_ENTRY.get().instance_id == instance_id {
FAST_ENTRY.set(FastEntry::EMPTY);
}
}