#![allow(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
clippy::cast_possible_wrap
)]
#![allow(unsafe_code)]
pub mod workloads;
use std::alloc::{GlobalAlloc, Layout, System};
use std::fmt;
use std::sync::atomic::{AtomicIsize, AtomicUsize, Ordering::Relaxed};
static LIVE: AtomicIsize = AtomicIsize::new(0);
static PEAK: AtomicIsize = AtomicIsize::new(0);
static TOTAL: AtomicUsize = AtomicUsize::new(0);
static COUNT: AtomicUsize = AtomicUsize::new(0);
static BUCKET_COUNT: [AtomicUsize; 48] = [const { AtomicUsize::new(0) }; 48];
static BUCKET_BYTES: [AtomicUsize; 48] = [const { AtomicUsize::new(0) }; 48];
pub struct Counting;
impl Default for Counting {
fn default() -> Self {
Self::new()
}
}
impl Counting {
pub const fn new() -> Self {
Counting
}
}
#[inline]
fn record(size: usize) {
TOTAL.fetch_add(size, Relaxed);
COUNT.fetch_add(1, Relaxed);
let k = ((usize::BITS - size.max(1).leading_zeros()) as usize - 1).min(47);
BUCKET_COUNT[k].fetch_add(1, Relaxed);
BUCKET_BYTES[k].fetch_add(size, Relaxed);
}
#[inline]
fn grew(by: usize) {
let by = by as isize;
let now = LIVE.fetch_add(by, Relaxed) + by;
PEAK.fetch_max(now, Relaxed);
}
#[inline]
fn shrank(by: usize) {
LIVE.fetch_sub(by as isize, Relaxed);
}
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
grew(l.size());
record(l.size());
unsafe { System.alloc(l) }
}
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
grew(l.size());
record(l.size());
unsafe { System.alloc_zeroed(l) }
}
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
shrank(l.size());
unsafe { System.dealloc(p, l) }
}
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
if new > l.size() {
grew(new - l.size());
record(new - l.size());
} else {
shrank(l.size() - new);
COUNT.fetch_add(1, Relaxed);
}
unsafe { System.realloc(p, l, new) }
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)]
pub struct Stats {
pub peak: usize,
pub total: usize,
pub allocs: usize,
}
impl Stats {
pub fn churn(&self) -> f64 {
self.total as f64 / (self.peak.max(1) as f64)
}
}
impl fmt::Display for Stats {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
const MB: f64 = 1048576.0;
write!(
f,
"peak {:>8.1} MB total {:>9.1} MB allocs {:>11} churn {:>4.1}x",
self.peak as f64 / MB,
self.total as f64 / MB,
self.allocs,
self.churn(),
)
}
}
pub fn measure<T>(f: impl FnOnce() -> T) -> (T, Stats) {
crate::clear_caches();
reset();
let out = f();
(out, snapshot())
}
pub fn reset() {
LIVE.store(0, Relaxed);
PEAK.store(0, Relaxed);
TOTAL.store(0, Relaxed);
COUNT.store(0, Relaxed);
for k in 0..48 {
BUCKET_COUNT[k].store(0, Relaxed);
BUCKET_BYTES[k].store(0, Relaxed);
}
}
pub fn snapshot() -> Stats {
Stats {
peak: PEAK.load(Relaxed).max(0) as usize,
total: TOTAL.load(Relaxed),
allocs: COUNT.load(Relaxed),
}
}
pub fn live() -> usize {
LIVE.load(Relaxed).max(0) as usize
}
pub fn histogram() -> Vec<(usize, usize, usize)> {
(0..48)
.filter_map(|k| {
let n = BUCKET_COUNT[k].load(Relaxed);
(n > 0).then(|| (1usize << k, n, BUCKET_BYTES[k].load(Relaxed)))
})
.collect()
}
#[derive(Clone, Copy, Debug)]
pub struct Budget {
pub name: &'static str,
pub peak: usize,
pub allocs: usize,
pub tolerance: f64,
}
impl Budget {
pub fn check(&self, stats: &Stats) -> Result<(), String> {
let scale = 1.0 + self.tolerance;
let peak_max = (self.peak as f64 * scale) as usize;
let allocs_max = (self.allocs as f64 * scale) as usize;
if stats.peak <= peak_max && stats.allocs <= allocs_max {
return Ok(());
}
const MB: f64 = 1048576.0;
Err(format!(
"memory budget exceeded for `{}`\n \
peak {:.1} MB (budget {:.1} MB, +{:.0}% = {:.1} MB)\n \
allocs {} (budget {}, +{:.0}% = {})\n \
If the increase is intended, update the budget to:\n \
Budget {{ name: \"{}\", peak: {}, allocs: {}, tolerance: {} }}",
self.name,
stats.peak as f64 / MB,
self.peak as f64 / MB,
self.tolerance * 100.0,
peak_max as f64 / MB,
stats.allocs,
self.allocs,
self.tolerance * 100.0,
allocs_max,
self.name,
stats.peak,
stats.allocs,
self.tolerance,
))
}
}