use core::ffi::c_void;
use core::fmt;
use core::ptr::NonNull;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArenaError {
Failed,
}
impl fmt::Display for ArenaError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Failed => f.write_str("mimalloc could not reserve or register the arena memory"),
}
}
}
impl core::error::Error for ArenaError {}
pub struct Heap {
ptr: NonNull<cyo_mimalloc_sys::mi_heap_t>,
owned: bool,
}
unsafe impl Send for Heap {}
unsafe impl Sync for Heap {}
impl Heap {
pub fn new() -> Option<Self> {
NonNull::new(unsafe { cyo_mimalloc_sys::mi_heap_new() }).map(Self::owned)
}
pub fn new_in_arena(arena_id: ArenaId) -> Option<Self> {
NonNull::new(unsafe { cyo_mimalloc_sys::mi_heap_new_in_arena(arena_id.0) }).map(Self::owned)
}
pub fn main() -> Self {
let ptr = unsafe { cyo_mimalloc_sys::mi_heap_main() };
Self::borrowed(NonNull::new(ptr).expect("mi_heap_main returned null"))
}
pub fn id(&self) -> HeapId {
HeapId(self.ptr.as_ptr() as usize)
}
pub unsafe fn contains(&self, ptr: *const u8) -> bool {
unsafe { cyo_mimalloc_sys::mi_heap_contains(self.ptr.as_ptr(), ptr as *const c_void) }
}
pub fn malloc(&self, size: usize) -> *mut u8 {
unsafe { cyo_mimalloc_sys::mi_heap_malloc(self.ptr.as_ptr(), size) as *mut u8 }
}
pub fn zalloc(&self, size: usize) -> *mut u8 {
unsafe { cyo_mimalloc_sys::mi_heap_zalloc(self.ptr.as_ptr(), size) as *mut u8 }
}
pub fn malloc_aligned(&self, size: usize, alignment: usize) -> *mut u8 {
unsafe {
cyo_mimalloc_sys::mi_heap_malloc_aligned(self.ptr.as_ptr(), size, alignment) as *mut u8
}
}
pub unsafe fn realloc(&self, ptr: *mut u8, new_size: usize) -> *mut u8 {
unsafe {
cyo_mimalloc_sys::mi_heap_realloc(self.ptr.as_ptr(), ptr as *mut c_void, new_size)
as *mut u8
}
}
pub fn delete(self) {
if self.owned {
unsafe { cyo_mimalloc_sys::mi_heap_delete(self.ptr.as_ptr()) }
}
core::mem::forget(self);
}
pub unsafe fn destroy(self) {
if self.owned {
unsafe { cyo_mimalloc_sys::mi_heap_destroy(self.ptr.as_ptr()) };
}
core::mem::forget(self);
}
pub fn collect(&self, force: bool) {
unsafe { cyo_mimalloc_sys::mi_heap_collect(self.ptr.as_ptr(), force) }
}
pub fn stats_json(&self, out: &mut (impl fmt::Write + ?Sized)) -> fmt::Result {
unsafe {
crate::ffi::write_owned_c_string(
out,
cyo_mimalloc_sys::mi_heap_stats_get_json(
self.ptr.as_ptr(),
0,
core::ptr::null_mut(),
),
)
}
}
pub fn stats_print(&self, out: &mut dyn fmt::Write) -> fmt::Result {
crate::ffi::write_output(out, |out, arg| unsafe {
cyo_mimalloc_sys::mi_heap_stats_print_out(self.ptr.as_ptr(), out, arg);
})
}
pub fn as_ptr(&self) -> *mut cyo_mimalloc_sys::mi_heap_t {
self.ptr.as_ptr()
}
#[inline]
fn owned(ptr: NonNull<cyo_mimalloc_sys::mi_heap_t>) -> Self {
Self { ptr, owned: true }
}
#[inline]
fn borrowed(ptr: NonNull<cyo_mimalloc_sys::mi_heap_t>) -> Self {
Self { ptr, owned: false }
}
}
impl Drop for Heap {
fn drop(&mut self) {
if self.owned {
unsafe { cyo_mimalloc_sys::mi_heap_delete(self.ptr.as_ptr()) }
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct HeapId(usize);
pub unsafe fn heap_of(ptr: *const u8) -> Option<HeapId> {
let heap = unsafe { cyo_mimalloc_sys::mi_heap_of(ptr as *const c_void) };
(!heap.is_null()).then_some(HeapId(heap as usize))
}
#[inline]
pub unsafe fn free(ptr: *mut u8) {
unsafe { cyo_mimalloc_sys::mi_free(ptr as *mut c_void) }
}
#[derive(Debug, Clone, Copy)]
pub struct ArenaId(cyo_mimalloc_sys::mi_arena_id_t);
unsafe impl Send for ArenaId {}
unsafe impl Sync for ArenaId {}
pub fn reserve_os_memory(
size: usize,
commit: bool,
allow_large: bool,
exclusive: bool,
) -> Result<ArenaId, ArenaError> {
let mut id = core::ptr::null_mut();
let rc = unsafe {
cyo_mimalloc_sys::mi_reserve_os_memory_ex(size, commit, allow_large, exclusive, &mut id)
};
if rc == 0 {
Ok(ArenaId(id))
} else {
Err(ArenaError::Failed)
}
}
pub fn reserve_huge_os_pages_interleave(
pages: usize,
numa_nodes: usize,
timeout_msecs: usize,
) -> Result<(), ArenaError> {
let rc = unsafe {
cyo_mimalloc_sys::mi_reserve_huge_os_pages_interleave(pages, numa_nodes, timeout_msecs)
};
if rc == 0 {
Ok(())
} else {
Err(ArenaError::Failed)
}
}
pub fn reserve_huge_os_pages_at(
pages: usize,
numa_node: i32,
timeout_msecs: usize,
) -> Result<(), ArenaError> {
let rc =
unsafe { cyo_mimalloc_sys::mi_reserve_huge_os_pages_at(pages, numa_node, timeout_msecs) };
if rc == 0 {
Ok(())
} else {
Err(ArenaError::Failed)
}
}
pub unsafe fn manage_os_memory(
start: *mut u8,
size: usize,
is_committed: bool,
is_pinned: bool,
is_zero: bool,
numa_node: i32,
exclusive: bool,
) -> Result<ArenaId, ArenaError> {
let mut id = core::ptr::null_mut();
let ok = unsafe {
cyo_mimalloc_sys::mi_manage_os_memory_ex(
start as *mut c_void,
size,
is_committed,
is_pinned,
is_zero,
numa_node,
exclusive,
&mut id,
)
};
if ok {
Ok(ArenaId(id))
} else {
Err(ArenaError::Failed)
}
}
#[inline]
pub fn arena_min_alignment() -> usize {
unsafe { cyo_mimalloc_sys::mi_arena_min_alignment() }
}
#[inline]
pub fn arena_min_size() -> usize {
unsafe { cyo_mimalloc_sys::mi_arena_min_size() }
}
#[inline]
pub fn arena_max_object_size() -> usize {
unsafe { cyo_mimalloc_sys::mi_arena_max_object_size() }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn heap_create_delete() {
let heap = Heap::new().expect("heap::new failed");
heap.delete();
}
#[test]
fn heap_alloc_free() {
let heap = Heap::new().unwrap();
let ptr = heap.malloc(128);
assert!(!ptr.is_null());
unsafe {
core::ptr::write_bytes(ptr, 0xCD, 128);
free(ptr);
}
heap.delete();
}
#[test]
fn heap_aligned_alloc() {
let heap = Heap::new().unwrap();
for pow in 0..=12 {
let align = 1usize << pow;
let ptr = heap.malloc_aligned(64, align);
assert!(!ptr.is_null());
assert_eq!(ptr as usize % align, 0, "align={align}");
unsafe { free(ptr) };
}
assert!(heap.malloc_aligned(64, 3).is_null());
heap.delete();
}
#[test]
fn heap_of_identifies_the_allocating_heap() {
let a = Heap::new().unwrap();
let b = Heap::new().unwrap();
let block = a.malloc(64);
unsafe {
assert_eq!(heap_of(block), Some(a.id()));
assert_ne!(heap_of(block), Some(b.id()));
assert!(a.contains(block));
assert!(!b.contains(block));
}
unsafe { free(block) };
}
#[test]
fn a_block_that_outlives_its_heap_belongs_to_the_main_heap() {
let heap = Heap::new().unwrap();
let block = heap.malloc(64);
drop(heap);
unsafe {
assert_eq!(heap_of(block), Some(Heap::main().id()));
free(block);
}
}
#[test]
fn free_accepts_null() {
unsafe { free(core::ptr::null_mut()) };
}
#[test]
fn arena_error_describes_itself() {
use std::string::ToString;
let error: &dyn core::error::Error = &ArenaError::Failed;
assert_eq!(
error.to_string(),
"mimalloc could not reserve or register the arena memory"
);
}
#[test]
fn arena_min_values_are_sane() {
let a = arena_min_alignment();
assert!(a > 0 && a.is_power_of_two());
}
#[test]
fn reserve_zero_huge_pages_is_ok() {
assert_eq!(reserve_huge_os_pages_interleave(0, 0, 0), Ok(()));
assert_eq!(reserve_huge_os_pages_at(0, 0, 0), Ok(()));
}
#[test]
fn heap_stats_are_available() {
let heap = Heap::new().unwrap();
let mut json = std::string::String::new();
heap.stats_json(&mut json).unwrap();
assert!(json.starts_with('{'));
let mut text = std::string::String::new();
heap.stats_print(&mut text).unwrap();
assert!(!text.is_empty());
heap.delete();
}
#[test]
fn borrowed_main_heap_delete_is_noop() {
let heap = Heap::main();
heap.delete();
unsafe {
let ptr = cyo_mimalloc_sys::mi_malloc(64);
assert!(!ptr.is_null());
cyo_mimalloc_sys::mi_free(ptr);
}
}
}