use core::{cell::OnceCell, cmp, sync::atomic::Ordering};
#[cfg(feature = "watermark")]
use portable_atomic::AtomicBool;
use portable_atomic::{AtomicU8, AtomicU64, AtomicUsize};
#[cfg(feature = "watermark")]
use crate::{Paint, Peak};
use crate::{Stack, config, platform};
pub const MAX_THREADS: usize = 256;
const EMPTY: u8 = 0;
const CLAIMING: u8 = 1;
const LIVE: u8 = 2;
const SCANNING: u8 = 3;
const DYING: u8 = 4;
#[cfg(feature = "histograms")]
#[repr(align(64))]
struct CacheAligned<T>(T);
#[repr(align(64))]
struct Control {
state: AtomicU8,
generation: AtomicU64,
}
impl Control {
const fn new() -> Self {
Self {
state: AtomicU8::new(EMPTY),
generation: AtomicU64::new(0),
}
}
}
const _: () = assert!(size_of::<Control>() == 64);
#[repr(align(64))]
struct Slot {
control: Control,
id: AtomicU64,
base: AtomicUsize,
limit: AtomicUsize,
#[cfg(feature = "watermark")]
paint_lo: AtomicUsize,
#[cfg(feature = "watermark")]
paint_hi: AtomicUsize,
#[cfg(feature = "watermark")]
saturated: AtomicBool,
depth_last: AtomicUsize,
peak: AtomicUsize,
sample_count: AtomicU64,
sample_sum: AtomicU64,
near_limit_count: AtomicU64,
guard_trip_count: AtomicU64,
#[cfg(feature = "histograms")]
buckets: [AtomicU64; 64],
}
impl Slot {
const fn new() -> Self {
Self {
control: Control::new(),
id: AtomicU64::new(0),
base: AtomicUsize::new(0),
limit: AtomicUsize::new(0),
#[cfg(feature = "watermark")]
paint_lo: AtomicUsize::new(0),
#[cfg(feature = "watermark")]
paint_hi: AtomicUsize::new(0),
#[cfg(feature = "watermark")]
saturated: AtomicBool::new(false),
depth_last: AtomicUsize::new(0),
peak: AtomicUsize::new(0),
sample_count: AtomicU64::new(0),
sample_sum: AtomicU64::new(0),
near_limit_count: AtomicU64::new(0),
guard_trip_count: AtomicU64::new(0),
#[cfg(feature = "histograms")]
buckets: [const { AtomicU64::new(0) }; 64],
}
}
fn stack(&self) -> Option<Stack> {
Stack::from_bounds(
self.limit.load(Ordering::Relaxed),
self.base.load(Ordering::Relaxed),
)
}
#[cfg(feature = "watermark")]
fn paint(&self) -> Option<Paint> {
Paint::from_bounds(
self.paint_lo.load(Ordering::Relaxed),
self.paint_hi.load(Ordering::Relaxed),
)
}
#[inline]
fn record(&self, depth: usize) {
self.peak.fetch_max(depth, Ordering::Relaxed);
self.depth_last.store(depth, Ordering::Relaxed);
self.sample_count.fetch_add(1, Ordering::Relaxed);
self.sample_sum.fetch_add(depth as u64, Ordering::Relaxed);
#[cfg(feature = "histograms")]
if depth > 0 {
self.buckets[crate::histogram::bucket_of(depth)].fetch_add(1, Ordering::Relaxed);
}
}
#[inline]
fn observe(&self, sp: usize) -> Option<usize> {
let base = self.base.load(Ordering::Relaxed);
let limit = self.limit.load(Ordering::Relaxed);
if sp >= base || sp < limit {
return None;
}
let depth = base - sp;
self.record(depth);
if sp - limit < config::reserve() {
self.near_limit_count.fetch_add(1, Ordering::Relaxed);
}
Some(depth)
}
fn snapshot(&self, idx: usize) -> Option<crate::ThreadStats> {
let generation = self.control.generation.load(Ordering::Acquire);
if !self.is_occupied() {
return None;
}
let stack = self.stack()?;
let depth_last = self.depth_last.load(Ordering::Relaxed);
let thread = crate::ThreadStats {
slot: idx,
id: self.id.load(Ordering::Relaxed),
base: stack.base(),
limit: stack.limit(),
size: stack.size(),
depth_last,
peak: cmp::max(self.peak.load(Ordering::Relaxed), depth_last),
#[cfg(feature = "watermark")]
saturated: self.saturated.load(Ordering::Relaxed),
sample_count: self.sample_count.load(Ordering::Relaxed),
sample_sum: self.sample_sum.load(Ordering::Relaxed),
near_limit_count: self.near_limit_count.load(Ordering::Relaxed),
guard_trip_count: self.guard_trip_count.load(Ordering::Relaxed),
};
core::sync::atomic::fence(Ordering::Acquire);
if self.control.generation.load(Ordering::Relaxed) != generation || !self.is_occupied() {
return None;
}
Some(thread)
}
fn is_occupied(&self) -> bool {
matches!(self.control.state.load(Ordering::Acquire), LIVE | SCANNING)
}
fn clear(&self) {
self.id.store(0, Ordering::Relaxed);
self.base.store(0, Ordering::Relaxed);
self.limit.store(0, Ordering::Relaxed);
#[cfg(feature = "watermark")]
{
self.paint_lo.store(0, Ordering::Relaxed);
self.paint_hi.store(0, Ordering::Relaxed);
self.saturated.store(false, Ordering::Relaxed);
}
self.depth_last.store(0, Ordering::Relaxed);
self.peak.store(0, Ordering::Relaxed);
self.sample_count.store(0, Ordering::Relaxed);
self.sample_sum.store(0, Ordering::Relaxed);
self.near_limit_count.store(0, Ordering::Relaxed);
self.guard_trip_count.store(0, Ordering::Relaxed);
#[cfg(feature = "histograms")]
for bucket in &self.buckets {
bucket.store(0, Ordering::Relaxed);
}
}
}
static SLOTS: [Slot; MAX_THREADS] = [const { Slot::new() }; MAX_THREADS];
#[repr(align(64))]
struct Retired {
peak_max: AtomicUsize,
sample_count: AtomicU64,
sample_sum: AtomicU64,
near_limit_count: AtomicU64,
guard_trip_count: AtomicU64,
saturated: AtomicU64,
}
static RETIRED: Retired = Retired {
peak_max: AtomicUsize::new(0),
sample_count: AtomicU64::new(0),
sample_sum: AtomicU64::new(0),
near_limit_count: AtomicU64::new(0),
guard_trip_count: AtomicU64::new(0),
saturated: AtomicU64::new(0),
};
static THREADS_SEEN: AtomicU64 = AtomicU64::new(0);
static UNTRACKED: AtomicU64 = AtomicU64::new(0);
#[cfg(feature = "histograms")]
static RETIRED_BUCKETS: CacheAligned<[AtomicU64; 64]> =
CacheAligned([const { AtomicU64::new(0) }; 64]);
struct Handle(usize);
impl Drop for Handle {
fn drop(&mut self) {
retire(self.0);
}
}
std::thread_local! {
static SLOT: OnceCell<Option<Handle>> = const { OnceCell::new() };
}
fn with_slot<R>(f: impl FnOnce(&Slot) -> R) -> Option<R> {
SLOT.try_with(|cell| {
cell.get_or_init(claim_slot)
.as_ref()
.map(|handle| f(&SLOTS[handle.0]))
})
.ok()
.flatten()
}
fn claim_slot() -> Option<Handle> {
let Some(stack) = Stack::current() else {
UNTRACKED.fetch_add(1, Ordering::Relaxed);
return None;
};
let id = platform::thread_id();
for (idx, slot) in SLOTS.iter().enumerate() {
if slot
.control
.state
.compare_exchange(EMPTY, CLAIMING, Ordering::Relaxed, Ordering::Relaxed)
.is_err()
{
continue;
}
slot.control.generation.fetch_add(1, Ordering::Release);
slot.id.store(id, Ordering::Relaxed);
slot.base.store(stack.base(), Ordering::Relaxed);
slot.limit.store(stack.limit(), Ordering::Relaxed);
#[cfg(feature = "watermark")]
if let Some(paint) = paint_current(&stack) {
slot.paint_lo.store(paint.lo(), Ordering::Relaxed);
slot.paint_hi.store(paint.hi(), Ordering::Relaxed);
}
slot.control.state.store(LIVE, Ordering::Release);
THREADS_SEEN.fetch_add(1, Ordering::Relaxed);
return Some(Handle(idx));
}
UNTRACKED.fetch_add(1, Ordering::Relaxed);
None
}
#[cfg(feature = "watermark")]
fn paint_current(stack: &Stack) -> Option<Paint> {
let depth = config::paint_depth();
if depth == 0 {
return None;
}
unsafe { stack.paint(depth) }
}
fn retire(idx: usize) {
let slot = &SLOTS[idx];
loop {
match slot
.control
.state
.compare_exchange(LIVE, DYING, Ordering::AcqRel, Ordering::Acquire)
{
Ok(_) => break,
Err(SCANNING) => std::thread::yield_now(),
Err(_) => return,
}
}
slot.control.generation.fetch_add(1, Ordering::Release);
#[cfg(feature = "watermark")]
{
scan(slot);
if slot.saturated.load(Ordering::Relaxed) {
RETIRED.saturated.fetch_add(1, Ordering::Relaxed);
}
}
RETIRED
.peak_max
.fetch_max(slot.peak.load(Ordering::Relaxed), Ordering::Relaxed);
RETIRED
.sample_count
.fetch_add(slot.sample_count.load(Ordering::Relaxed), Ordering::Relaxed);
RETIRED
.sample_sum
.fetch_add(slot.sample_sum.load(Ordering::Relaxed), Ordering::Relaxed);
RETIRED.near_limit_count.fetch_add(
slot.near_limit_count.load(Ordering::Relaxed),
Ordering::Relaxed,
);
RETIRED.guard_trip_count.fetch_add(
slot.guard_trip_count.load(Ordering::Relaxed),
Ordering::Relaxed,
);
#[cfg(feature = "histograms")]
for (retired, mine) in RETIRED_BUCKETS.0.iter().zip(&slot.buckets) {
retired.fetch_add(mine.load(Ordering::Relaxed), Ordering::Relaxed);
}
slot.clear();
slot.control.state.store(EMPTY, Ordering::Release);
}
#[cfg(feature = "watermark")]
fn scan(slot: &Slot) -> Option<Peak> {
let stack = slot.stack()?;
let paint = slot.paint()?;
let peak = unsafe { stack.peak(&paint) };
slot.peak.fetch_max(peak.bytes(), Ordering::Relaxed);
if peak.is_saturated() {
slot.saturated.store(true, Ordering::Relaxed);
}
Some(peak)
}
#[cfg_attr(feature = "watermark", doc = "")]
#[cfg_attr(
feature = "watermark",
doc = "[`peak`] and [`repaint`] attach on first use as well."
)]
pub fn attach() -> bool {
with_slot(|_| ()).is_some()
}
pub fn is_attached() -> bool {
SLOT.try_with(|cell| matches!(cell.get(), Some(Some(_))))
.unwrap_or(false)
}
pub fn current_stack() -> Option<Stack> {
with_slot(Slot::stack).flatten()
}
pub fn depth() -> Option<usize> {
let sp = platform::sp();
with_slot(|slot| slot.stack()?.depth_at(sp)).flatten()
}
pub fn headroom() -> Option<usize> {
let sp = platform::sp();
with_slot(|slot| {
let stack = slot.stack()?;
if stack.contains(sp) {
Some(sp - stack.limit())
} else {
None
}
})
.flatten()
}
pub fn has_headroom(bytes: usize) -> bool {
let sp = platform::sp();
with_slot(|slot| {
let limit = slot.limit.load(Ordering::Relaxed);
if sp < limit || sp - limit >= bytes {
return true;
}
slot.guard_trip_count.fetch_add(1, Ordering::Relaxed);
false
})
.unwrap_or(true)
}
pub fn sample() -> Option<usize> {
let sp = platform::sp();
with_slot(|slot| slot.observe(sp)).flatten()
}
#[cfg(feature = "watermark")]
pub fn peak() -> Option<Peak> {
with_slot(scan).flatten()
}
#[cfg(feature = "watermark")]
pub fn repaint() -> Option<Paint> {
with_slot(|slot| {
scan(slot);
let stack = slot.stack()?;
let paint = paint_current(&stack)?;
slot.paint_lo.store(paint.lo(), Ordering::Relaxed);
slot.paint_hi.store(paint.hi(), Ordering::Relaxed);
Some(paint)
})
.flatten()
}
#[cfg(feature = "watermark")]
pub(crate) fn record_peak(depth: usize, saturated: bool) {
with_slot(|slot| {
slot.peak.fetch_max(depth, Ordering::Relaxed);
if saturated {
slot.saturated.store(true, Ordering::Relaxed);
}
});
}
#[cfg_attr(feature = "sampler", doc = "[sampler](crate::sampler)")]
#[cfg_attr(not(feature = "sampler"), doc = "sampler")]
#[cfg(feature = "watermark")]
pub fn poll_all() {
for slot in &SLOTS {
if slot
.control
.state
.compare_exchange(LIVE, SCANNING, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
continue;
}
scan(slot);
slot.control.state.store(LIVE, Ordering::Release);
}
}
pub fn for_each_thread(mut f: impl FnMut(crate::ThreadStats)) {
for (idx, slot) in SLOTS.iter().enumerate() {
if let Some(thread) = slot.snapshot(idx) {
f(thread);
}
}
}
pub fn threads() -> Vec<crate::ThreadStats> {
let mut out = Vec::new();
for_each_thread(|thread| out.push(thread));
out
}
pub(crate) struct RetiredTotals {
pub(crate) peak_max: usize,
pub(crate) sample_count: u64,
pub(crate) sample_sum: u64,
pub(crate) near_limit_count: u64,
pub(crate) guard_trip_count: u64,
#[cfg(feature = "watermark")]
pub(crate) saturated: u64,
}
pub(crate) fn retired_totals() -> RetiredTotals {
RetiredTotals {
peak_max: RETIRED.peak_max.load(Ordering::Relaxed),
sample_count: RETIRED.sample_count.load(Ordering::Relaxed),
sample_sum: RETIRED.sample_sum.load(Ordering::Relaxed),
near_limit_count: RETIRED.near_limit_count.load(Ordering::Relaxed),
guard_trip_count: RETIRED.guard_trip_count.load(Ordering::Relaxed),
#[cfg(feature = "watermark")]
saturated: RETIRED.saturated.load(Ordering::Relaxed),
}
}
pub(crate) fn threads_seen() -> u64 {
THREADS_SEEN.load(Ordering::Relaxed)
}
pub(crate) fn untracked() -> u64 {
UNTRACKED.load(Ordering::Relaxed)
}
#[cfg(feature = "histograms")]
pub(crate) fn depth_histogram() -> crate::Histogram {
let mut buckets = [0u64; 64];
for (bucket, retired) in buckets.iter_mut().zip(&RETIRED_BUCKETS.0) {
*bucket = retired.load(Ordering::Relaxed);
}
for slot in SLOTS.iter().filter(|slot| slot.is_occupied()) {
for (bucket, mine) in buckets.iter_mut().zip(&slot.buckets) {
*bucket += mine.load(Ordering::Relaxed);
}
}
crate::Histogram::from_buckets(buckets)
}
#[cfg_attr(
feature = "watermark",
doc = ", or [`crate::measure_peak`], which repaints."
)]
#[cfg_attr(not(feature = "watermark"), doc = ".")]
pub fn reset() {
RETIRED.peak_max.store(0, Ordering::Relaxed);
RETIRED.sample_count.store(0, Ordering::Relaxed);
RETIRED.sample_sum.store(0, Ordering::Relaxed);
RETIRED.near_limit_count.store(0, Ordering::Relaxed);
RETIRED.guard_trip_count.store(0, Ordering::Relaxed);
RETIRED.saturated.store(0, Ordering::Relaxed);
let observed = THREADS_SEEN.load(Ordering::Relaxed);
let _ =
THREADS_SEEN.compare_exchange(observed, count_live(), Ordering::Relaxed, Ordering::Relaxed);
UNTRACKED.store(0, Ordering::Relaxed);
#[cfg(feature = "histograms")]
for bucket in &RETIRED_BUCKETS.0 {
bucket.store(0, Ordering::Relaxed);
}
for slot in &SLOTS {
if !slot.is_occupied() {
continue;
}
#[cfg(feature = "histograms")]
for bucket in &slot.buckets {
bucket.store(0, Ordering::Relaxed);
}
slot.sample_count.store(0, Ordering::Relaxed);
slot.sample_sum.store(0, Ordering::Relaxed);
slot.near_limit_count.store(0, Ordering::Relaxed);
slot.guard_trip_count.store(0, Ordering::Relaxed);
let observed = slot.peak.load(Ordering::Relaxed);
let _ = slot.peak.compare_exchange(
observed,
slot.depth_last.load(Ordering::Relaxed),
Ordering::Relaxed,
Ordering::Relaxed,
);
#[cfg(feature = "watermark")]
slot.saturated.store(false, Ordering::Relaxed);
}
}
fn count_live() -> u64 {
SLOTS.iter().filter(|slot| slot.is_occupied()).count() as u64
}