use std::alloc::{GlobalAlloc, Layout, System};
use std::cell::Cell;
use std::ptr::NonNull;
use std::sync::OnceLock;
use crate::{
BoAllocError, BoAllocator, Config, InvariantError, StatsSnapshot, TrimMode, TrimStats,
};
thread_local! {
static IN_GLOBAL_ALLOCATOR: Cell<bool> = const { Cell::new(false) };
}
struct GlobalCallGuard;
impl GlobalCallGuard {
#[inline]
fn enter() -> Option<Self> {
IN_GLOBAL_ALLOCATOR
.try_with(|active| {
if active.replace(true) {
None
} else {
Some(Self)
}
})
.ok()
.flatten()
}
}
impl Drop for GlobalCallGuard {
#[inline]
fn drop(&mut self) {
let _ = IN_GLOBAL_ALLOCATOR.try_with(|active| active.set(false));
}
}
pub struct BoGlobalAllocator {
config: Config,
allocator: OnceLock<Result<BoAllocator, BoAllocError>>,
}
impl BoGlobalAllocator {
pub const fn new() -> Self {
Self::with_config(Config::new())
}
pub const fn with_config(config: Config) -> Self {
Self {
config,
allocator: OnceLock::new(),
}
}
pub fn initialize(&self) -> Result<(), BoAllocError> {
self.with_allocator(|_| ())
}
pub fn stats(&self) -> Result<StatsSnapshot, BoAllocError> {
self.with_allocator(BoAllocator::stats)
}
pub fn trim(&self, mode: TrimMode) -> Result<TrimStats, BoAllocError> {
self.with_allocator(|allocator| allocator.trim(mode))
}
pub fn validate(&self) -> Result<(), InvariantError> {
const MESSAGE_CAPACITY: usize = 1024;
struct FixedMessage {
bytes: [u8; MESSAGE_CAPACITY],
len: usize,
}
impl FixedMessage {
fn from_invariant(error: &InvariantError) -> Self {
let source = error.0.as_bytes();
let mut len = source.len().min(MESSAGE_CAPACITY);
while !error.0.is_char_boundary(len) {
len -= 1;
}
let mut bytes = [0; MESSAGE_CAPACITY];
bytes[..len].copy_from_slice(&source[..len]);
Self { bytes, len }
}
}
struct PendingError {
initialization: Option<BoAllocError>,
invariant: FixedMessage,
}
let validation = {
let Some(_guard) = GlobalCallGuard::enter() else {
return Err(InvariantError::new(
"global allocator validation was called re-entrantly",
));
};
match self.initialize_guarded() {
Err(error) => Err(PendingError {
initialization: Some(error),
invariant: FixedMessage {
bytes: [0; MESSAGE_CAPACITY],
len: 0,
},
}),
Ok(allocator) => allocator.validate().map_err(|error| PendingError {
initialization: None,
invariant: FixedMessage::from_invariant(&error),
}),
}
};
match validation {
Ok(()) => Ok(()),
Err(error) if error.initialization.is_some() => {
let error = error.initialization.expect("checked above");
Err(InvariantError::new(format!(
"global allocator initialization failed: {error}"
)))
}
Err(error) => {
let text = std::str::from_utf8(&error.invariant.bytes[..error.invariant.len])
.expect("copied invariant message remains UTF-8");
Err(InvariantError::new(text))
}
}
}
fn with_allocator<R>(
&self,
operation: impl FnOnce(&BoAllocator) -> R,
) -> Result<R, BoAllocError> {
if let Some(result) = self.allocator.get() {
let Some(_guard) = GlobalCallGuard::enter() else {
return Err(BoAllocError::OutOfMemory);
};
return result.as_ref().map(operation).map_err(|error| *error);
}
let Some(_guard) = GlobalCallGuard::enter() else {
return Err(BoAllocError::OutOfMemory);
};
self.initialize_guarded().map(operation)
}
#[inline]
fn initialize_guarded(&self) -> Result<&BoAllocator, BoAllocError> {
self.allocator
.get_or_init(|| BoAllocator::new(self.config.clone()))
.as_ref()
.map_err(|error| *error)
}
#[inline]
fn initialized_owner(&self, address: usize) -> Option<&BoAllocator> {
match self.allocator.get() {
Some(Ok(allocator))
if !crate::tls::thread_exiting() || allocator.owns_address(address) =>
{
Some(allocator)
}
_ => None,
}
}
#[inline]
unsafe fn allocate_guarded(&self, layout: Layout, zeroed: bool) -> *mut u8 {
if !crate::allocator::thread_cache_available() {
if zeroed {
return unsafe { System.alloc_zeroed(layout) };
}
return unsafe { System.alloc(layout) };
}
match self.initialize_guarded() {
Ok(allocator) => match allocator.allocate(layout) {
Ok(allocation) => {
let pointer = allocation.as_ptr() as *mut u8;
if zeroed && layout.size() != 0 {
unsafe { pointer.write_bytes(0, layout.size()) };
}
pointer
}
Err(_) => std::ptr::null_mut(),
},
Err(_) => {
if zeroed {
unsafe { System.alloc_zeroed(layout) }
} else {
unsafe { System.alloc(layout) }
}
}
}
}
}
impl Default for BoGlobalAllocator {
fn default() -> Self {
Self::new()
}
}
unsafe impl GlobalAlloc for BoGlobalAllocator {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
if layout.size() == 0 {
return layout.align() as *mut u8;
}
let Some(_guard) = GlobalCallGuard::enter() else {
return unsafe { System.alloc(layout) };
};
unsafe { self.allocate_guarded(layout, false) }
}
#[inline]
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
if layout.size() == 0 {
return layout.align() as *mut u8;
}
let Some(_guard) = GlobalCallGuard::enter() else {
return unsafe { System.alloc_zeroed(layout) };
};
unsafe { self.allocate_guarded(layout, true) }
}
#[inline]
unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) {
if pointer.is_null() || layout.size() == 0 {
return;
}
let Some(_guard) = GlobalCallGuard::enter() else {
unsafe { System.dealloc(pointer, layout) };
return;
};
if let Some(allocator) = self.initialized_owner(pointer as usize) {
unsafe { allocator.deallocate(NonNull::new_unchecked(pointer), layout) };
} else {
unsafe { System.dealloc(pointer, layout) };
}
}
#[inline]
unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
if pointer.is_null() {
let Ok(new_layout) = Layout::from_size_align(new_size, layout.align()) else {
return std::ptr::null_mut();
};
return unsafe { self.alloc(new_layout) };
}
if new_size == 0 {
unsafe { self.dealloc(pointer, layout) };
return std::ptr::null_mut();
}
let Some(_guard) = GlobalCallGuard::enter() else {
return unsafe { System.realloc(pointer, layout, new_size) };
};
if self.initialized_owner(pointer as usize).is_none() {
return unsafe { System.realloc(pointer, layout, new_size) };
}
unsafe { self.reallocate_bomalloc(pointer, layout, new_size) }
}
}
impl BoGlobalAllocator {
#[inline]
unsafe fn reallocate_bomalloc(
&self,
pointer: *mut u8,
layout: Layout,
new_size: usize,
) -> *mut u8 {
let Some(allocator) = self.initialized_owner(pointer as usize) else {
return std::ptr::null_mut();
};
let Ok(new_layout) = Layout::from_size_align(new_size, layout.align()) else {
return std::ptr::null_mut();
};
let new_pointer = if crate::allocator::thread_cache_available() {
let Ok(allocation) = allocator.allocate(new_layout) else {
return std::ptr::null_mut();
};
allocation.as_ptr() as *mut u8
} else {
unsafe { System.alloc(new_layout) }
};
if new_pointer.is_null() {
return std::ptr::null_mut();
}
unsafe {
std::ptr::copy_nonoverlapping(pointer, new_pointer, layout.size().min(new_size));
allocator.deallocate(NonNull::new_unchecked(pointer), layout);
}
new_pointer
}
}