use ::std::alloc::{GlobalAlloc, Layout, System};
use ::std::cell::{Cell, UnsafeCell};
use ::std::hint;
use ::std::mem::MaybeUninit;
use ::std::ptr::{self, NonNull};
use ::std::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering};
use ::rlsf::Tlsf;
pub const POOL_SIZE: usize = 4 << 20;
type Pool = Tlsf<'static, u32, u32, 20, 32>;
thread_local! {
static AUDIO_PATH: Cell<bool> = const { Cell::new(false) };
}
pub fn audio_path<R>(f: impl FnOnce() -> R) -> R {
struct Restore(bool);
impl Drop for Restore {
fn drop(&mut self) {
AUDIO_PATH.set(self.0);
}
}
let _restore = Restore(AUDIO_PATH.replace(true));
f()
}
fn in_audio_path() -> bool {
AUDIO_PATH.get()
}
pub struct RtAlloc {
memory: UnsafeCell<MaybeUninit<[u8; POOL_SIZE]>>,
pool: UnsafeCell<Pool>,
ready: UnsafeCell<bool>,
locked: AtomicBool,
used: AtomicUsize,
peak: AtomicUsize,
outside: AtomicU64,
}
unsafe impl Sync for RtAlloc {}
static ACTIVE: AtomicPtr<RtAlloc> = AtomicPtr::new(ptr::null_mut());
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PoolStats {
pub size: usize,
pub used: usize,
pub peak: usize,
pub outside: u64,
}
struct Locked<'a>(&'a RtAlloc);
impl Locked<'_> {
fn pool(&mut self) -> &mut Pool {
let alloc = self.0;
unsafe {
if !*alloc.ready.get() {
alloc.prepare();
*alloc.ready.get() = true;
}
&mut *alloc.pool.get()
}
}
}
impl Drop for Locked<'_> {
fn drop(&mut self) {
self.0.locked.store(false, Ordering::Release);
}
}
impl RtAlloc {
#[allow(clippy::new_without_default)] pub const fn new() -> Self {
Self {
memory: UnsafeCell::new(MaybeUninit::uninit()),
pool: UnsafeCell::new(Tlsf::new()),
ready: UnsafeCell::new(false),
locked: AtomicBool::new(false),
used: AtomicUsize::new(0),
peak: AtomicUsize::new(0),
outside: AtomicU64::new(0),
}
}
pub fn init(&'static self) {
self.lock().pool();
ACTIVE.store(ptr::from_ref(self).cast_mut(), Ordering::Release);
}
pub fn stats(&self) -> PoolStats {
PoolStats {
size: POOL_SIZE,
used: self.used.load(Ordering::Relaxed),
peak: self.peak.load(Ordering::Relaxed),
outside: self.outside.load(Ordering::Relaxed),
}
}
pub fn contains(&self, ptr: *const u8) -> bool {
(ptr as usize).wrapping_sub(self.memory.get() as usize) < POOL_SIZE
}
fn lock(&self) -> Locked<'_> {
while self
.locked
.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
hint::spin_loop();
}
Locked(self)
}
unsafe fn prepare(&self) {
let memory = self.memory.get().cast::<u8>();
unsafe {
for offset in (0..POOL_SIZE).step_by(4096) {
memory.add(offset).write_volatile(0);
}
let block = NonNull::slice_from_raw_parts(NonNull::new_unchecked(memory), POOL_SIZE);
(*self.pool.get()).insert_free_block_ptr(block);
}
}
fn pool_alloc(&self, layout: Layout) -> Option<*mut u8> {
let ptr = self.lock().pool().allocate(layout)?;
let used = self.used.fetch_add(layout.size(), Ordering::Relaxed) + layout.size();
self.peak.fetch_max(used, Ordering::Relaxed);
Some(ptr.as_ptr())
}
unsafe fn pool_dealloc(&self, ptr: *mut u8, layout: Layout) {
unsafe {
self.lock()
.pool()
.deallocate(NonNull::new_unchecked(ptr), layout.align())
};
self.used.fetch_sub(layout.size(), Ordering::Relaxed);
}
fn count_outside(&self) {
self.outside.fetch_add(1, Ordering::Relaxed);
}
}
pub fn stats() -> Option<PoolStats> {
unsafe { ACTIVE.load(Ordering::Acquire).as_ref() }.map(RtAlloc::stats)
}
unsafe impl GlobalAlloc for RtAlloc {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
if in_audio_path() {
if let Some(ptr) = self.pool_alloc(layout) {
return ptr;
}
self.count_outside();
}
unsafe { System.alloc(layout) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
if in_audio_path() {
if let Some(ptr) = self.pool_alloc(layout) {
unsafe { ptr.write_bytes(0, layout.size()) };
return ptr;
}
self.count_outside();
}
unsafe { System.alloc_zeroed(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
if self.contains(ptr) {
unsafe { self.pool_dealloc(ptr, layout) };
} else {
if in_audio_path() {
self.count_outside();
}
unsafe { System.dealloc(ptr, layout) };
}
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
let audio = in_audio_path();
let from_pool = self.contains(ptr);
if !from_pool && !audio {
return unsafe { System.realloc(ptr, layout, new_size) };
}
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
if from_pool && audio {
let moved = unsafe {
self.lock()
.pool()
.reallocate(NonNull::new_unchecked(ptr), new_layout)
};
if let Some(moved) = moved {
self.used.fetch_sub(layout.size(), Ordering::Relaxed);
let used = self.used.fetch_add(new_size, Ordering::Relaxed) + new_size;
self.peak.fetch_max(used, Ordering::Relaxed);
return moved.as_ptr();
}
}
let new = unsafe { self.alloc(new_layout) };
if !new.is_null() {
unsafe {
ptr::copy_nonoverlapping(ptr, new, layout.size().min(new_size));
self.dealloc(ptr, layout);
}
}
new
}
}