bomalloc 0.3.0

A fragmentation-aware experimental allocator built around address-span ownership
Documentation
use std::alloc::{GlobalAlloc, Layout, System};
use std::cell::Cell;
use std::ptr::NonNull;
use std::sync::OnceLock;

thread_local! {
    // This flag has no destructor and is therefore safe to consult from a
    // process-global allocator and from native TLS cleanup callbacks.
    static THREAD_EXITING: Cell<bool> = const { Cell::new(false) };
}

#[inline]
pub(crate) fn thread_exiting() -> bool {
    THREAD_EXITING.try_with(Cell::get).unwrap_or(true)
}

#[derive(Clone, Copy, Debug)]
pub(crate) struct AccessError;

/// Destructor-capable TLS that does not use Rust std's allocating destructor
/// registry. Allocator TLS must use native FLS/pthread keys because registering
/// a Rust `thread_local!` destructor can recursively invoke the global allocator.
pub(crate) struct NativeThreadLocal<T: 'static> {
    key: OnceLock<usize>,
    initialize: fn() -> T,
}

#[repr(C)]
struct Slot<T> {
    key: usize,
    value: T,
}

// Each value is reachable only through the current OS thread's key.
unsafe impl<T: 'static> Sync for NativeThreadLocal<T> {}

impl<T: 'static> NativeThreadLocal<T> {
    pub const fn new(initialize: fn() -> T) -> Self {
        Self {
            key: OnceLock::new(),
            initialize,
        }
    }

    pub fn with<R>(&self, use_value: impl FnOnce(&T) -> R) -> R {
        self.try_with(use_value)
            .expect("cannot access allocator TLS during thread destruction")
    }

    pub fn try_with<R>(&self, use_value: impl FnOnce(&T) -> R) -> Result<R, AccessError> {
        let exiting = thread_exiting();
        let key = if exiting {
            *self.key.get().ok_or(AccessError)?
        } else {
            *self.key.get_or_init(native::allocate_key::<T>)
        };
        if native::key_failed(key) {
            return Err(AccessError);
        }

        let mut pointer = unsafe { native::get_value(key) }.cast::<Slot<T>>();
        if pointer.is_null() {
            if exiting {
                return Err(AccessError);
            }
            let layout = Layout::new::<Slot<T>>();
            // Use System directly: allocating storage for allocator TLS through
            // the process allocator would recursively initialize this same key.
            pointer = unsafe { System.alloc(layout) }.cast::<Slot<T>>();
            let Some(non_null) = NonNull::new(pointer) else {
                return Err(AccessError);
            };
            unsafe {
                non_null.as_ptr().write(Slot {
                    key,
                    value: (self.initialize)(),
                })
            };
            if !unsafe { native::set_value(key, non_null.as_ptr().cast()) } {
                unsafe {
                    std::ptr::drop_in_place(non_null.as_ptr());
                    System.dealloc(non_null.as_ptr().cast(), layout);
                }
                return Err(AccessError);
            }
        }

        // SAFETY: the native key owns one initialized T until its current-thread
        // destructor runs; calls on one key are confined to that thread.
        Ok(use_value(unsafe { &(*pointer).value }))
    }
}

unsafe fn destroy<T>(pointer: *mut Slot<T>) {
    if pointer.is_null() {
        return;
    }
    // Windows may leave the FLS value visible while invoking its callback.
    // Clear it before any field destructor can allocate or free through the
    // process allocator and attempt to consult this key again.
    let key = unsafe { (*pointer).key };
    let _ = unsafe { native::set_value(key, std::ptr::null_mut()) };
    let _ = THREAD_EXITING.try_with(|exiting| exiting.set(true));
    unsafe {
        std::ptr::drop_in_place(pointer);
        System.dealloc(pointer.cast(), Layout::new::<Slot<T>>());
    }
}

#[cfg(windows)]
mod native {
    use std::ffi::c_void;

    use windows_sys::Win32::System::Threading::{
        FlsAlloc, FlsGetValue, FlsSetValue, FLS_OUT_OF_INDEXES,
    };

    pub(super) fn allocate_key<T: 'static>() -> usize {
        unsafe extern "system" fn callback<T: 'static>(pointer: *const c_void) {
            unsafe { super::destroy(pointer.cast_mut().cast::<super::Slot<T>>()) };
        }

        unsafe { FlsAlloc(Some(callback::<T>)) as usize }
    }

    pub(super) fn key_failed(key: usize) -> bool {
        key == FLS_OUT_OF_INDEXES as usize
    }

    pub(super) unsafe fn get_value(key: usize) -> *mut c_void {
        unsafe { FlsGetValue(key as u32) }
    }

    pub(super) unsafe fn set_value(key: usize, pointer: *mut c_void) -> bool {
        unsafe { FlsSetValue(key as u32, pointer.cast_const()) != 0 }
    }
}

#[cfg(target_os = "linux")]
mod native {
    use std::ffi::c_void;
    use std::mem::MaybeUninit;

    const FAILED_KEY: usize = usize::MAX;

    pub(super) fn allocate_key<T: 'static>() -> usize {
        unsafe extern "C" fn callback<T: 'static>(pointer: *mut c_void) {
            unsafe { super::destroy(pointer.cast::<super::Slot<T>>()) };
        }

        let mut key = MaybeUninit::<libc::pthread_key_t>::uninit();
        let result = unsafe { libc::pthread_key_create(key.as_mut_ptr(), Some(callback::<T>)) };
        if result == 0 {
            unsafe { key.assume_init() as usize }
        } else {
            FAILED_KEY
        }
    }

    pub(super) fn key_failed(key: usize) -> bool {
        key == FAILED_KEY
    }

    pub(super) unsafe fn get_value(key: usize) -> *mut c_void {
        unsafe { libc::pthread_getspecific(key as libc::pthread_key_t) }
    }

    pub(super) unsafe fn set_value(key: usize, pointer: *mut c_void) -> bool {
        unsafe { libc::pthread_setspecific(key as libc::pthread_key_t, pointer) == 0 }
    }
}

#[cfg(not(any(windows, target_os = "linux")))]
mod native {
    use std::ffi::c_void;

    pub(super) fn allocate_key<T: 'static>() -> usize {
        let _ = std::marker::PhantomData::<T>;
        usize::MAX
    }

    pub(super) fn key_failed(_key: usize) -> bool {
        true
    }

    pub(super) unsafe fn get_value(_key: usize) -> *mut c_void {
        std::ptr::null_mut()
    }

    pub(super) unsafe fn set_value(_key: usize, _pointer: *mut c_void) -> bool {
        false
    }
}