use std::{
alloc::{GlobalAlloc, Layout, System},
cell::Cell,
sync::atomic::{AtomicBool, AtomicUsize, Ordering},
};
use super::AllocationSnapshot;
pub struct CountingAllocator<A = System> {
inner: A,
}
impl CountingAllocator<System> {
pub const fn system() -> Self {
Self::new(System)
}
}
impl<A> CountingAllocator<A> {
pub const fn new(inner: A) -> Self {
Self { inner }
}
}
static INSTALLED: AtomicBool = AtomicBool::new(false);
const SHARDS: usize = 16;
#[repr(align(128))]
struct Shard(AtomicUsize);
static LIVE_BYTES: [Shard; SHARDS] = [const { Shard(AtomicUsize::new(0)) }; SHARDS];
static NEXT_SHARD: AtomicUsize = AtomicUsize::new(0);
struct ThreadCounts {
allocations: Cell<u64>,
frees: Cell<u64>,
allocated_bytes: Cell<u64>,
freed_bytes: Cell<u64>,
shard: Cell<usize>,
}
thread_local! {
static THREAD: ThreadCounts = const {
ThreadCounts {
allocations: Cell::new(0),
frees: Cell::new(0),
allocated_bytes: Cell::new(0),
freed_bytes: Cell::new(0),
shard: Cell::new(SHARDS),
}
};
}
#[inline]
fn add(cell: &Cell<u64>, value: u64) {
cell.set(cell.get().wrapping_add(value));
}
impl ThreadCounts {
#[inline]
fn shard(&self) -> &'static AtomicUsize {
let mut shard = self.shard.get();
if shard == SHARDS {
shard = NEXT_SHARD.fetch_add(1, Ordering::Relaxed) % SHARDS;
self.shard.set(shard);
}
&LIVE_BYTES[shard].0
}
#[inline]
fn allocated(&self, size: usize) {
add(&self.allocations, 1);
add(&self.allocated_bytes, size as u64);
}
#[inline]
fn freed(&self, size: usize) {
add(&self.frees, 1);
add(&self.freed_bytes, size as u64);
}
}
#[inline]
fn note(allocated: Option<usize>, freed: Option<usize>) {
if allocated.is_some() && !INSTALLED.load(Ordering::Relaxed) {
INSTALLED.store(true, Ordering::Relaxed);
}
let change = allocated.unwrap_or(0).wrapping_sub(freed.unwrap_or(0));
let counted = THREAD.try_with(|counts| {
if let Some(size) = allocated {
counts.allocated(size);
}
if let Some(size) = freed {
counts.freed(size);
}
counts.shard().fetch_add(change, Ordering::Relaxed);
});
if counted.is_err() {
LIVE_BYTES[0].0.fetch_add(change, Ordering::Relaxed);
}
}
unsafe impl<A: GlobalAlloc> GlobalAlloc for CountingAllocator<A> {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let ptr = unsafe { self.inner.alloc(layout) };
if !ptr.is_null() {
note(Some(layout.size()), None);
}
ptr
}
#[inline]
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
let ptr = unsafe { self.inner.alloc_zeroed(layout) };
if !ptr.is_null() {
note(Some(layout.size()), None);
}
ptr
}
#[inline]
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe { self.inner.dealloc(ptr, layout) };
note(None, Some(layout.size()));
}
#[inline]
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
let new = unsafe { self.inner.realloc(ptr, layout, new_size) };
if !new.is_null() {
note(Some(new_size), Some(layout.size()));
}
new
}
}
pub(crate) fn snapshot() -> Option<AllocationSnapshot> {
if !INSTALLED.load(Ordering::Relaxed) {
return None;
}
let live_bytes = LIVE_BYTES
.iter()
.fold(0_usize, |sum, shard| sum.wrapping_add(shard.0.load(Ordering::Relaxed)));
THREAD
.try_with(|counts| AllocationSnapshot {
allocations: counts.allocations.get(),
frees: counts.frees.get(),
allocated_bytes: counts.allocated_bytes.get(),
freed_bytes: counts.freed_bytes.get(),
live_bytes: (live_bytes as isize).max(0) as u64,
})
.ok()
}