use std::alloc::Layout;
use std::ffi::c_void;
use std::mem::{align_of, size_of};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::ptr::NonNull;
use std::sync::OnceLock;
use crate::{BoAllocError, BoAllocator, Config, StatsSnapshot, TrimMode, TrimStats};
const ABI_VERSION: u32 = 1;
const C_ALIGNMENT: usize = 16;
const HEADER_ALIGNMENT_SHIFT: u32 = usize::BITS - 8;
const HEADER_SIZE_MASK: usize = usize::MAX >> 8;
const BO_OK: i32 = 0;
const BO_ENOMEM: i32 = 12;
const BO_EINVAL: i32 = 22;
const BO_EUNSUPPORTED: i32 = 1001;
const BO_EINVARIANT: i32 = 1002;
const BO_EPANIC: i32 = 1003;
const BO_TRIM_EXCESS: u32 = 0;
const BO_TRIM_AGGRESSIVE: u32 = 1;
static C_ALLOCATOR: OnceLock<Result<BoAllocator, BoAllocError>> = OnceLock::new();
#[inline]
fn c_allocator() -> Result<&'static BoAllocator, BoAllocError> {
C_ALLOCATOR
.get_or_init(|| BoAllocator::new(Config::new()))
.as_ref()
.map_err(|error| *error)
}
#[repr(C)]
#[derive(Clone, Copy)]
struct CAllocationHeader {
requested_and_alignment: usize,
}
const _: () = assert!(size_of::<CAllocationHeader>() == size_of::<usize>());
impl CAllocationHeader {
fn encode_request(size: usize, alignment: usize) -> Option<usize> {
if size > HEADER_SIZE_MASK {
return None;
}
Some(size | ((alignment.trailing_zeros() as usize) << HEADER_ALIGNMENT_SHIFT))
}
fn requested_size(self) -> usize {
self.requested_and_alignment & HEADER_SIZE_MASK
}
fn backing_align(self) -> Option<usize> {
1usize.checked_shl((self.requested_and_alignment >> HEADER_ALIGNMENT_SHIFT) as u32)
}
}
struct DecodedCAllocation {
header: CAllocationHeader,
base: *mut u8,
backing_size: usize,
backing_align: usize,
user_offset: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct BoCStatsV1 {
pub abi_version: u32,
pub struct_size: u32,
pub reserved_bytes: u64,
pub committed_bytes: u64,
pub requested_live_bytes: u64,
pub usable_live_bytes: u64,
pub metadata_bytes: u64,
pub c_header_bytes_per_allocation: u64,
pub c_default_alignment: u64,
pub green_spans: u64,
pub white_spans: u64,
pub red_spans: u64,
pub black_spans: u64,
pub splits: u64,
pub merges: u64,
pub steals: u64,
pub cas_failures: u64,
pub random_jumps: u64,
pub decommits: u64,
pub chunk_releases: u64,
}
impl BoCStatsV1 {
fn from_snapshot(snapshot: StatsSnapshot) -> Self {
Self {
abi_version: ABI_VERSION,
struct_size: size_of::<Self>() as u32,
reserved_bytes: snapshot.reserved_bytes as u64,
committed_bytes: snapshot.committed_bytes as u64,
requested_live_bytes: snapshot.requested_live_bytes as u64,
usable_live_bytes: snapshot.usable_live_bytes as u64,
metadata_bytes: snapshot.metadata_bytes as u64,
c_header_bytes_per_allocation: size_of::<CAllocationHeader>() as u64,
c_default_alignment: C_ALIGNMENT as u64,
green_spans: snapshot.green_spans as u64,
white_spans: snapshot.white_spans as u64,
red_spans: snapshot.red_spans as u64,
black_spans: snapshot.black_spans as u64,
splits: snapshot.splits,
merges: snapshot.merges,
steals: snapshot.steals,
cas_failures: snapshot.cas_failures,
random_jumps: snapshot.random_jumps,
decommits: snapshot.decommits,
chunk_releases: snapshot.chunk_releases,
}
}
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct BoCTrimStatsV1 {
pub abi_version: u32,
pub struct_size: u32,
pub spans_reclaimed: u64,
pub bytes_decommitted: u64,
pub chunks_released: u64,
pub bytes_released: u64,
}
impl From<TrimStats> for BoCTrimStatsV1 {
fn from(stats: TrimStats) -> Self {
Self {
abi_version: ABI_VERSION,
struct_size: size_of::<Self>() as u32,
spans_reclaimed: stats.spans_reclaimed as u64,
bytes_decommitted: stats.bytes_decommitted as u64,
chunks_released: stats.chunks_released as u64,
bytes_released: stats.bytes_released as u64,
}
}
}
#[inline]
fn catch_or<T>(fallback: T, operation: impl FnOnce() -> T) -> T {
catch_unwind(AssertUnwindSafe(operation)).unwrap_or(fallback)
}
#[inline]
fn error_code(error: BoAllocError) -> i32 {
match error {
BoAllocError::OutOfMemory | BoAllocError::AddressSpaceExhausted => BO_ENOMEM,
BoAllocError::InvalidConfig => BO_EINVAL,
BoAllocError::UnsupportedPlatform => BO_EUNSUPPORTED,
}
}
#[inline]
fn align_up(value: usize, alignment: usize) -> Option<usize> {
value
.checked_add(alignment - 1)
.map(|sum| sum & !(alignment - 1))
}
#[inline(always)]
fn allocate_c_default(size: usize, zeroed: bool) -> *mut c_void {
if size > HEADER_SIZE_MASK {
return std::ptr::null_mut();
}
let payload_size = size.max(1);
let Some(backing_size) = C_ALIGNMENT.checked_add(payload_size) else {
return std::ptr::null_mut();
};
let layout = Layout::from_size_align(backing_size, C_ALIGNMENT)
.expect("the encoded C size fits a default-aligned Layout");
let Ok(allocator) = c_allocator() else {
return std::ptr::null_mut();
};
let Ok(allocation) = allocator.allocate(layout) else {
return std::ptr::null_mut();
};
let base = allocation.as_ptr() as *mut u8;
let user = unsafe { base.add(C_ALIGNMENT) };
let requested_and_alignment =
size | ((C_ALIGNMENT.trailing_zeros() as usize) << HEADER_ALIGNMENT_SHIFT);
unsafe {
user.sub(size_of::<CAllocationHeader>())
.cast::<CAllocationHeader>()
.write(CAllocationHeader {
requested_and_alignment,
});
if zeroed && size != 0 {
user.write_bytes(0, size);
}
}
user.cast()
}
fn allocate_c(size: usize, alignment: usize, zeroed: bool) -> *mut c_void {
if !alignment.is_power_of_two() {
return std::ptr::null_mut();
}
let effective_alignment = alignment
.max(C_ALIGNMENT)
.max(align_of::<CAllocationHeader>());
if effective_alignment == C_ALIGNMENT {
return allocate_c_default(size, zeroed);
}
let Some(requested_and_alignment) =
CAllocationHeader::encode_request(size, effective_alignment)
else {
return std::ptr::null_mut();
};
let payload_size = size.max(1);
let user_offset = effective_alignment;
let Some(backing_size) = user_offset.checked_add(payload_size) else {
return std::ptr::null_mut();
};
let Ok(layout) = Layout::from_size_align(backing_size, effective_alignment) else {
return std::ptr::null_mut();
};
let Ok(allocator) = c_allocator() else {
return std::ptr::null_mut();
};
let Ok(allocation) = allocator.allocate(layout) else {
return std::ptr::null_mut();
};
let base = allocation.as_ptr() as *mut u8;
let Some(user_address) = (base as usize).checked_add(user_offset) else {
unsafe { allocator.deallocate(NonNull::new_unchecked(base), layout) };
return std::ptr::null_mut();
};
let user = user_address as *mut u8;
let header_pointer =
unsafe { user.sub(size_of::<CAllocationHeader>()) }.cast::<CAllocationHeader>();
unsafe {
header_pointer.write(CAllocationHeader {
requested_and_alignment,
});
if zeroed && size != 0 {
user.write_bytes(0, size);
}
}
user.cast()
}
unsafe fn header(pointer: *mut c_void) -> Option<DecodedCAllocation> {
let user = pointer.cast::<u8>();
let header_pointer =
unsafe { user.sub(size_of::<CAllocationHeader>()) }.cast::<CAllocationHeader>();
let non_null = NonNull::new(header_pointer)?;
let header = unsafe { non_null.as_ptr().read() };
let requested_size = header.requested_size();
if header.requested_and_alignment >> HEADER_ALIGNMENT_SHIFT
== C_ALIGNMENT.trailing_zeros() as usize
{
let backing_size = C_ALIGNMENT.checked_add(requested_size.max(1))?;
return Some(DecodedCAllocation {
header,
base: unsafe { user.sub(C_ALIGNMENT) },
backing_size,
backing_align: C_ALIGNMENT,
user_offset: C_ALIGNMENT,
});
}
let backing_align = header.backing_align()?;
if backing_align < C_ALIGNMENT || backing_align < align_of::<CAllocationHeader>() {
return None;
}
let user_offset = align_up(size_of::<CAllocationHeader>(), backing_align)?;
let backing_size = user_offset.checked_add(requested_size.max(1))?;
let base_address = (user as usize).checked_sub(user_offset)?;
if base_address == 0
|| base_address & (backing_align - 1) != 0
|| user as usize & (backing_align - 1) != 0
{
return None;
}
Some(DecodedCAllocation {
header,
base: base_address as *mut u8,
backing_size,
backing_align,
user_offset,
})
}
unsafe fn free_c(pointer: *mut c_void) {
if pointer.is_null() {
return;
}
let Some(allocation) = (unsafe { header(pointer) }) else {
debug_assert!(false, "pointer was not allocated by bomalloc's C ABI");
return;
};
let Ok(layout) = Layout::from_size_align(allocation.backing_size, allocation.backing_align)
else {
return;
};
let Ok(allocator) = c_allocator() else {
return;
};
unsafe { allocator.deallocate(NonNull::new_unchecked(allocation.base), layout) };
}
unsafe fn realloc_c(pointer: *mut c_void, new_size: usize) -> *mut c_void {
if pointer.is_null() {
return allocate_c(new_size, C_ALIGNMENT, false);
}
if new_size == 0 {
unsafe { free_c(pointer) };
return std::ptr::null_mut();
}
let Some(old) = (unsafe { header(pointer) }) else {
return std::ptr::null_mut();
};
let replacement = allocate_c(new_size, old.backing_align, false);
if replacement.is_null() {
return std::ptr::null_mut();
}
unsafe {
std::ptr::copy_nonoverlapping(
pointer.cast::<u8>(),
replacement.cast::<u8>(),
old.header.requested_size().min(new_size),
);
free_c(pointer);
}
replacement
}
#[unsafe(no_mangle)]
pub extern "C" fn bo_allocator_abi_version() -> u32 {
ABI_VERSION
}
#[unsafe(no_mangle)]
pub extern "C" fn bo_allocator_initialize() -> i32 {
catch_or(BO_EPANIC, || match c_allocator() {
Ok(_) => BO_OK,
Err(error) => error_code(error),
})
}
#[unsafe(no_mangle)]
pub extern "C" fn bo_malloc(size: usize) -> *mut c_void {
allocate_c(size, C_ALIGNMENT, false)
}
#[unsafe(no_mangle)]
pub extern "C" fn bo_calloc(count: usize, size: usize) -> *mut c_void {
count
.checked_mul(size)
.map_or(std::ptr::null_mut(), |total| {
allocate_c(total, C_ALIGNMENT, true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn bo_aligned_alloc(alignment: usize, size: usize) -> *mut c_void {
if alignment < size_of::<*mut c_void>() || !alignment.is_power_of_two() || size % alignment != 0
{
std::ptr::null_mut()
} else {
allocate_c(size, alignment, false)
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_posix_memalign(
output: *mut *mut c_void,
alignment: usize,
size: usize,
) -> i32 {
if output.is_null()
|| alignment < size_of::<*mut c_void>()
|| alignment % size_of::<*mut c_void>() != 0
|| !alignment.is_power_of_two()
{
return BO_EINVAL;
}
let pointer = allocate_c(size, alignment, false);
if pointer.is_null() {
BO_ENOMEM
} else {
unsafe { output.write(pointer) };
BO_OK
}
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_free(pointer: *mut c_void) {
unsafe { free_c(pointer) };
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_realloc(pointer: *mut c_void, new_size: usize) -> *mut c_void {
unsafe { realloc_c(pointer, new_size) }
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_reallocarray(
pointer: *mut c_void,
count: usize,
size: usize,
) -> *mut c_void {
count
.checked_mul(size)
.map_or(std::ptr::null_mut(), |total| unsafe {
realloc_c(pointer, total)
})
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_malloc_usable_size(pointer: *mut c_void) -> usize {
if pointer.is_null() {
return 0;
}
(unsafe { header(pointer) }).map_or(0, |allocation| {
allocation.backing_size - allocation.user_offset
})
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_allocator_stats_v1(output: *mut BoCStatsV1) -> i32 {
catch_or(BO_EPANIC, || {
if output.is_null() {
return BO_EINVAL;
}
match c_allocator() {
Ok(allocator) => {
let snapshot = allocator.stats();
unsafe { output.write(BoCStatsV1::from_snapshot(snapshot)) };
BO_OK
}
Err(error) => error_code(error),
}
})
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn bo_allocator_trim_v1(mode: u32, output: *mut BoCTrimStatsV1) -> i32 {
catch_or(BO_EPANIC, || {
if output.is_null() {
return BO_EINVAL;
}
let mode = match mode {
BO_TRIM_EXCESS => TrimMode::Excess,
BO_TRIM_AGGRESSIVE => TrimMode::Aggressive,
_ => return BO_EINVAL,
};
match c_allocator() {
Ok(allocator) => {
let stats = allocator.trim(mode);
unsafe { output.write(stats.into()) };
BO_OK
}
Err(error) => error_code(error),
}
})
}
#[unsafe(no_mangle)]
pub extern "C" fn bo_allocator_validate() -> i32 {
catch_or(BO_EPANIC, || match c_allocator() {
Ok(allocator) => match allocator.validate() {
Ok(()) => BO_OK,
Err(_) => BO_EINVARIANT,
},
Err(error) => error_code(error),
})
}
#[cfg(test)]
mod tests {
use std::sync::mpsc;
use std::thread;
use super::*;
#[test]
fn malloc_calloc_and_free_round_trip() {
for size in [0, 1, 16, 96, 4096, 96 * 1024, 2 * 1024 * 1024] {
let pointer = bo_malloc(size);
assert!(!pointer.is_null());
assert_eq!(pointer as usize % C_ALIGNMENT, 0);
assert!(unsafe { bo_malloc_usable_size(pointer) } >= size.max(1));
unsafe {
pointer.cast::<u8>().write_bytes(0xa5, size);
bo_free(pointer);
}
}
let pointer = bo_calloc(257, 3);
assert!(!pointer.is_null());
assert!((0..771).all(|index| unsafe { pointer.cast::<u8>().add(index).read() == 0 }));
unsafe { bo_free(pointer) };
assert!(bo_calloc(usize::MAX, 2).is_null());
}
#[test]
fn aligned_and_posix_allocations_validate_arguments() {
let aligned_64 = bo_aligned_alloc(64, 128);
assert!(!aligned_64.is_null());
assert_eq!(aligned_64 as usize % 64, 0);
unsafe { bo_free(aligned_64) };
let aligned = bo_aligned_alloc(4096, 8192);
assert!(!aligned.is_null());
assert_eq!(aligned as usize % 4096, 0);
unsafe { bo_free(aligned) };
assert!(bo_aligned_alloc(24, 96).is_null());
assert!(bo_aligned_alloc(64, 65).is_null());
let mut pointer = std::ptr::dangling_mut::<c_void>();
assert_eq!(unsafe { bo_posix_memalign(&mut pointer, 256, 513) }, BO_OK);
assert_eq!(pointer as usize % 256, 0);
unsafe { bo_free(pointer) };
pointer = std::ptr::dangling_mut();
assert_eq!(
unsafe { bo_posix_memalign(&mut pointer, 24, 100) },
BO_EINVAL
);
assert_eq!(pointer, std::ptr::dangling_mut());
}
#[test]
fn realloc_and_reallocarray_preserve_contents() {
let pointer = bo_malloc(257);
assert!(!pointer.is_null());
for index in 0..257 {
unsafe { pointer.cast::<u8>().add(index).write((index % 251) as u8) };
}
let grown = unsafe { bo_realloc(pointer, 96 * 1024) };
assert!(!grown.is_null());
assert!(
(0..257).all(|index| unsafe {
grown.cast::<u8>().add(index).read() == (index % 251) as u8
})
);
let failed = unsafe { bo_reallocarray(grown, usize::MAX, 2) };
assert!(failed.is_null());
assert!(
(0..257).all(|index| unsafe {
grown.cast::<u8>().add(index).read() == (index % 251) as u8
})
);
let shrunk = unsafe { bo_realloc(grown, 64) };
assert!(!shrunk.is_null());
assert!(
(0..64).all(|index| unsafe {
shrunk.cast::<u8>().add(index).read() == (index % 251) as u8
})
);
assert!(unsafe { bo_realloc(shrunk, 0) }.is_null());
}
#[test]
fn allocations_can_cross_threads_and_management_api_is_versioned() {
assert_eq!(bo_allocator_abi_version(), ABI_VERSION);
assert_eq!(bo_allocator_initialize(), BO_OK);
let (sender, receiver) = mpsc::channel();
let producer = thread::spawn(move || {
let pointer = bo_malloc(1024);
assert!(!pointer.is_null());
unsafe { pointer.cast::<u8>().write_bytes(0x3c, 1024) };
sender.send(pointer as usize).unwrap();
});
let pointer = receiver.recv().unwrap() as *mut c_void;
producer.join().unwrap();
assert!((0..1024).all(|index| unsafe { pointer.cast::<u8>().add(index).read() == 0x3c }));
let mut stats = BoCStatsV1::default();
assert_eq!(unsafe { bo_allocator_stats_v1(&mut stats) }, BO_OK);
assert_eq!(stats.abi_version, ABI_VERSION);
assert_eq!(stats.struct_size as usize, size_of::<BoCStatsV1>());
assert_eq!(
stats.c_header_bytes_per_allocation,
size_of::<CAllocationHeader>() as u64
);
assert_eq!(stats.c_default_alignment, C_ALIGNMENT as u64);
unsafe { bo_free(pointer) };
let mut trimmed = BoCTrimStatsV1::default();
assert_eq!(
unsafe { bo_allocator_trim_v1(BO_TRIM_EXCESS, &mut trimmed) },
BO_OK
);
assert_eq!(trimmed.abi_version, ABI_VERSION);
assert_eq!(trimmed.struct_size as usize, size_of::<BoCTrimStatsV1>());
assert_eq!(bo_allocator_validate(), BO_OK);
assert_eq!(unsafe { bo_allocator_trim_v1(99, &mut trimmed) }, BO_EINVAL);
}
}