Skip to main content

libc_rs/
heap.rs

1//! Dynamic `mmap_anonymous` slab and page-granularity allocator for unikernel EL0 binaries.
2
3use core::cell::UnsafeCell;
4use core::ptr::null_mut;
5use core::sync::atomic::{AtomicBool, Ordering};
6
7use crate::sync::FutexMutex;
8
9#[repr(C)]
10struct FreeNode {
11    next: *mut FreeNode,
12}
13
14#[inline]
15const fn align_up(value: usize, align: usize) -> Option<usize> {
16    let mask = align - 1;
17    match value.checked_add(mask) {
18        Some(v) => Some(v & !mask),
19        None => None,
20    }
21}
22
23const MMAP_NUM_CLASSES: usize = 8;
24const _: () = assert!(MMAP_NUM_CLASSES == 8);
25const MMAP_CLASS_TOTAL_SIZES: [usize; MMAP_NUM_CLASSES] = [32, 64, 128, 256, 512, 1024, 2048, 4096];
26const MMAP_HEADER_BYTES: usize = 16;
27const MMAP_LARGE_HEADER_BYTES: usize = 32;
28const MMAP_MAX_SMALL_PAYLOAD: usize = 4080;
29const MMAP_SLAB_BYTES: usize = 16 * crate::mm::PAGE_SIZE;
30
31const SMALL_MAGIC_BASE: usize = 0xC0DE_5A1B_0000_0000;
32const SMALL_MAGIC_MASK: usize = 0xFFFF_FFFF_FFFF_FF00;
33const LARGE_MAGIC: usize = 0xC0DE_1A6E_5A1B_0001;
34
35const MAGAZINE_SLOTS: usize = 64;
36const MAGAZINE_CAP: usize = 16;
37const MAGAZINE_BATCH: usize = 8;
38const _: () = {
39    assert!(MAGAZINE_SLOTS == 64);
40    assert!(MAGAZINE_CAP == 16);
41};
42
43#[derive(Copy, Clone)]
44struct ClassMagazine {
45    len: usize,
46    ptrs: [*mut u8; 16],
47}
48
49impl ClassMagazine {
50    const fn new() -> Self {
51        Self {
52            len: 0,
53            ptrs: [null_mut(); 16],
54        }
55    }
56}
57
58#[repr(align(64))]
59struct ThreadMagazineSlot {
60    in_use: AtomicBool,
61    classes: UnsafeCell<[ClassMagazine; 8]>,
62}
63
64// SAFETY: Access to `classes` is serialized per slot via `in_use` acquire/release compare-exchange.
65unsafe impl Sync for ThreadMagazineSlot {}
66
67impl ThreadMagazineSlot {
68    const fn new() -> Self {
69        Self {
70            in_use: AtomicBool::new(false),
71            classes: UnsafeCell::new([ClassMagazine::new(); 8]),
72        }
73    }
74}
75
76#[inline]
77fn current_magazine_slot_index() -> usize {
78    let tid = crate::thread::current_thread_id();
79    let mixed = tid ^ (tid >> 6) ^ (tid >> 12) ^ (tid >> 19);
80    (mixed as usize) & (MAGAZINE_SLOTS - 1)
81}
82
83#[inline]
84const fn mmap_small_class_index(size: usize) -> Option<usize> {
85    let total = match size.checked_add(MMAP_HEADER_BYTES) {
86        Some(v) => v,
87        None => return None,
88    };
89    let mut i = 0;
90    while i < MMAP_NUM_CLASSES {
91        if total <= MMAP_CLASS_TOTAL_SIZES[i] {
92            return Some(i);
93        }
94        i += 1;
95    }
96    None
97}
98
99struct MmapHeapInner {
100    free_lists: [*mut FreeNode; 8],
101    slab_ptr: *mut u8,
102    slab_rem: usize,
103}
104
105// SAFETY: `MmapHeapInner` is only accessed while holding `MmapHeap::inner`'s `FutexMutex`.
106unsafe impl Send for MmapHeapInner {}
107
108impl MmapHeapInner {
109    const fn new() -> Self {
110        Self {
111            free_lists: [
112                null_mut(),
113                null_mut(),
114                null_mut(),
115                null_mut(),
116                null_mut(),
117                null_mut(),
118                null_mut(),
119                null_mut(),
120            ],
121            slab_ptr: null_mut(),
122            slab_rem: 0,
123        }
124    }
125
126    unsafe fn alloc_small(&mut self, class_idx: usize) -> *mut u8 {
127        let block_bytes = MMAP_CLASS_TOTAL_SIZES[class_idx];
128        let payload_bytes = block_bytes - MMAP_HEADER_BYTES;
129        let head = self.free_lists[class_idx];
130        let block_ptr = if !head.is_null() {
131            // SAFETY: `head` is a valid free-list node previously carved from a slab.
132            unsafe {
133                self.free_lists[class_idx] = (*head).next;
134            }
135            head.cast::<u8>()
136        } else {
137            if self.slab_rem < block_bytes {
138                let slab = match crate::mm::mmap_anonymous(
139                    MMAP_SLAB_BYTES,
140                    crate::mm::PROT_READ | crate::mm::PROT_WRITE,
141                    0,
142                ) {
143                    Ok(p) if !p.is_null() => p,
144                    _ => return null_mut(),
145                };
146                self.slab_ptr = slab;
147                self.slab_rem = MMAP_SLAB_BYTES;
148            }
149            let p = self.slab_ptr;
150            // SAFETY: `self.slab_rem >= block_bytes` guarantees `add(block_bytes)` stays within the slab.
151            self.slab_ptr = unsafe { self.slab_ptr.add(block_bytes) };
152            self.slab_rem -= block_bytes;
153            p
154        };
155
156        // SAFETY: `block_ptr` is 16-byte aligned and has at least `block_bytes` writable bytes.
157        unsafe {
158            let hdr = block_ptr.cast::<usize>();
159            hdr.write(SMALL_MAGIC_BASE | class_idx);
160            hdr.add(1).write(payload_bytes);
161            block_ptr.add(MMAP_HEADER_BYTES)
162        }
163    }
164
165    unsafe fn free_small(&mut self, block_ptr: *mut u8, class_idx: usize) {
166        let node = block_ptr.cast::<FreeNode>();
167        // SAFETY: `block_ptr` is the start of a small slab block of size >= 32 bytes.
168        unsafe {
169            (*node).next = self.free_lists[class_idx];
170        }
171        self.free_lists[class_idx] = node;
172    }
173}
174
175static GLOBAL_MAGAZINES: [ThreadMagazineSlot; 64] = [const { ThreadMagazineSlot::new() }; 64];
176
177/// Dynamic virtual-memory allocator backed by `mmap_anonymous` slabs for small blocks
178/// (`<= 4080` bytes) and direct page-granularity `mmap_anonymous`/`munmap` mappings for
179/// large or over-aligned allocations.
180pub struct MmapHeap {
181    inner: FutexMutex<MmapHeapInner>,
182    use_tls_cache: bool,
183}
184
185impl Default for MmapHeap {
186    fn default() -> Self {
187        Self::new()
188    }
189}
190
191impl MmapHeap {
192    /// Creates a new uninitialized [`MmapHeap`] without the per-thread magazine cache.
193    #[must_use]
194    pub const fn new() -> Self {
195        Self {
196            inner: FutexMutex::new(MmapHeapInner::new()),
197            use_tls_cache: false,
198        }
199    }
200
201    /// Creates a new uninitialized [`MmapHeap`] with the lock-free per-thread small-block magazine cache enabled.
202    #[must_use]
203    pub const fn new_cached() -> Self {
204        Self {
205            inner: FutexMutex::new(MmapHeapInner::new()),
206            use_tls_cache: true,
207        }
208    }
209
210    #[inline]
211    unsafe fn alloc_small_cached(&self, class_idx: usize) -> *mut u8 {
212        let slot = &GLOBAL_MAGAZINES[current_magazine_slot_index()];
213        if slot
214            .in_use
215            .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
216            .is_ok()
217        {
218            // SAFETY: `slot.in_use` is exclusively held by the current thread.
219            let mag = unsafe { &mut (*slot.classes.get())[class_idx] };
220            if mag.len > 0 {
221                mag.len -= 1;
222                let ptr = mag.ptrs[mag.len];
223                slot.in_use.store(false, Ordering::Release);
224                return ptr;
225            }
226
227            let mut inner = self.inner.lock();
228            // SAFETY: `class_idx` is a valid small class index.
229            let first = unsafe { inner.alloc_small(class_idx) };
230            if !first.is_null() {
231                let mut i = 1;
232                while i < MAGAZINE_BATCH && mag.len < MAGAZINE_CAP {
233                    // SAFETY: `class_idx` is a valid small class index.
234                    let extra = unsafe { inner.alloc_small(class_idx) };
235                    if extra.is_null() {
236                        break;
237                    }
238                    mag.ptrs[mag.len] = extra;
239                    mag.len += 1;
240                    i += 1;
241                }
242            }
243            drop(inner);
244            slot.in_use.store(false, Ordering::Release);
245            return first;
246        }
247
248        let mut inner = self.inner.lock();
249        // SAFETY: `class_idx` is a valid small class index.
250        unsafe { inner.alloc_small(class_idx) }
251    }
252
253    #[inline]
254    unsafe fn free_small_cached(&self, ptr: *mut u8, block_ptr: *mut u8, class_idx: usize) {
255        let slot = &GLOBAL_MAGAZINES[current_magazine_slot_index()];
256        if slot
257            .in_use
258            .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
259            .is_ok()
260        {
261            // SAFETY: `slot.in_use` is exclusively held by the current thread.
262            let mag = unsafe { &mut (*slot.classes.get())[class_idx] };
263            if mag.len < MAGAZINE_CAP {
264                mag.ptrs[mag.len] = ptr;
265                mag.len += 1;
266                slot.in_use.store(false, Ordering::Release);
267                return;
268            }
269
270            let mut inner = self.inner.lock();
271            let mut spilled = 0;
272            while spilled < MAGAZINE_BATCH && mag.len > 0 {
273                mag.len -= 1;
274                let victim_ptr = mag.ptrs[mag.len];
275                // SAFETY: `victim_ptr` is a small user pointer preceded by `MMAP_HEADER_BYTES`.
276                let victim_block = unsafe { victim_ptr.sub(MMAP_HEADER_BYTES) };
277                unsafe { inner.free_small(victim_block, class_idx) };
278                spilled += 1;
279            }
280            drop(inner);
281            mag.ptrs[mag.len] = ptr;
282            mag.len += 1;
283            slot.in_use.store(false, Ordering::Release);
284            return;
285        }
286
287        let mut inner = self.inner.lock();
288        // SAFETY: `block_ptr` is a small slab block for `class_idx`.
289        unsafe { inner.free_small(block_ptr, class_idx) };
290    }
291
292    /// Allocates `size` bytes with alignment `align` (at least 16-byte aligned), recording
293    /// C-compatible metadata immediately preceding the returned user pointer.
294    ///
295    /// # Safety
296    ///
297    /// Any non-null pointer returned must be released via [`MmapHeap::free_c`] or [`MmapHeap::realloc_c`].
298    pub unsafe fn alloc_c(&self, size: usize, align: usize) -> *mut u8 {
299        let eff_align = if align < 16 { 16 } else { align };
300        if !eff_align.is_power_of_two() {
301            return null_mut();
302        }
303        let req_size = if size == 0 { 1 } else { size };
304
305        if eff_align == 16 && req_size <= MMAP_MAX_SMALL_PAYLOAD {
306            if let Some(class_idx) = mmap_small_class_index(req_size) {
307                if self.use_tls_cache {
308                    // SAFETY: `class_idx` is a valid small class index.
309                    return unsafe { self.alloc_small_cached(class_idx) };
310                }
311                let mut inner = self.inner.lock();
312                // SAFETY: `class_idx` is a valid small class index.
313                return unsafe { inner.alloc_small(class_idx) };
314            }
315        }
316
317        // SAFETY: `eff_align` is a power of two >= 16.
318        unsafe { Self::alloc_large(req_size, eff_align) }
319    }
320
321    unsafe fn alloc_large(size: usize, align: usize) -> *mut u8 {
322        let extra = match align.checked_add(MMAP_LARGE_HEADER_BYTES) {
323            Some(v) => v,
324            None => return null_mut(),
325        };
326        let raw_needed = match size.checked_add(extra) {
327            Some(v) => v,
328            None => return null_mut(),
329        };
330        let map_total_len = match align_up(raw_needed, crate::mm::PAGE_SIZE) {
331            Some(v) => v,
332            None => return null_mut(),
333        };
334
335        let map_base = match crate::mm::mmap_anonymous(
336            map_total_len,
337            crate::mm::PROT_READ | crate::mm::PROT_WRITE,
338            0,
339        ) {
340            Ok(p) if !p.is_null() => p,
341            _ => return null_mut(),
342        };
343
344        let base_addr = map_base as usize;
345        let min_user_addr = match base_addr.checked_add(MMAP_LARGE_HEADER_BYTES) {
346            Some(v) => v,
347            None => {
348                // SAFETY: `map_base` and `map_total_len` were just returned by `mmap_anonymous`.
349                let _ = unsafe { crate::mm::munmap(map_base, map_total_len) };
350                return null_mut();
351            }
352        };
353        let user_addr = match align_up(min_user_addr, align) {
354            Some(v) => v,
355            None => {
356                // SAFETY: `map_base` and `map_total_len` were just returned by `mmap_anonymous`.
357                let _ = unsafe { crate::mm::munmap(map_base, map_total_len) };
358                return null_mut();
359            }
360        };
361        let offset = user_addr - base_addr;
362        let usable_size = map_total_len.saturating_sub(offset);
363        // SAFETY: `user_addr` lies within `[base_addr + 32, base_addr + map_total_len)`.
364        unsafe {
365            let user_ptr = map_base.add(offset);
366            let hdr = user_ptr.sub(MMAP_LARGE_HEADER_BYTES).cast::<usize>();
367            hdr.write(base_addr);
368            hdr.add(1).write(map_total_len);
369            hdr.add(2).write(LARGE_MAGIC);
370            hdr.add(3).write(usable_size);
371            user_ptr
372        }
373    }
374
375    /// Allocates and zero-initializes `nobj * size` bytes.
376    ///
377    /// # Safety
378    ///
379    /// Any non-null pointer returned must be released via [`MmapHeap::free_c`] or [`MmapHeap::realloc_c`].
380    pub unsafe fn calloc_c(&self, nobj: usize, size: usize) -> *mut u8 {
381        let total = match nobj.checked_mul(size) {
382            Some(v) => v,
383            None => return null_mut(),
384        };
385        let req_size = if total == 0 { 1 } else { total };
386        if req_size <= MMAP_MAX_SMALL_PAYLOAD {
387            // SAFETY: `req_size` is validated and alignment is 16.
388            let ptr = unsafe { self.alloc_c(req_size, 16) };
389            if !ptr.is_null() {
390                // SAFETY: `ptr` has at least `req_size` writable bytes.
391                unsafe {
392                    crate::mem::set_bytes(ptr, 0, req_size);
393                }
394            }
395            return ptr;
396        }
397        // SAFETY: `req_size > MMAP_MAX_SMALL_PAYLOAD` takes the direct `mmap_anonymous` path.
398        unsafe { Self::alloc_large(req_size, 16) }
399    }
400
401    /// Frees a pointer previously returned by [`MmapHeap::alloc_c`], [`MmapHeap::calloc_c`],
402    /// or [`MmapHeap::realloc_c`]. Passing null is a safe no-op.
403    ///
404    /// # Safety
405    ///
406    /// `ptr` must be null or a live pointer allocated by this [`MmapHeap`].
407    pub unsafe fn free_c(&self, ptr: *mut u8) {
408        if ptr.is_null() {
409            return;
410        }
411        // SAFETY: Every pointer allocated by `MmapHeap` is preceded by at least 16 bytes of metadata.
412        let hdr = unsafe { ptr.sub(MMAP_HEADER_BYTES).cast::<usize>() };
413        let magic = unsafe { hdr.read() };
414
415        if (magic & SMALL_MAGIC_MASK) == SMALL_MAGIC_BASE {
416            let class_idx = magic & 0xFF;
417            if class_idx < MMAP_NUM_CLASSES {
418                let block_ptr = hdr.cast::<u8>();
419                if self.use_tls_cache {
420                    // SAFETY: `ptr` and `block_ptr` belong to small slab class `class_idx`.
421                    unsafe { self.free_small_cached(ptr, block_ptr, class_idx) };
422                    return;
423                }
424                let mut inner = self.inner.lock();
425                // SAFETY: `block_ptr` is a small slab block for `class_idx`.
426                unsafe { inner.free_small(block_ptr, class_idx) };
427            }
428            return;
429        }
430
431        if magic == LARGE_MAGIC {
432            // SAFETY: Large allocations store `[base_addr, map_total_len]` in the 16 bytes before `hdr`.
433            unsafe {
434                let large_hdr = ptr.sub(MMAP_LARGE_HEADER_BYTES).cast::<usize>();
435                let base_addr = large_hdr.read();
436                let map_total_len = large_hdr.add(1).read();
437                let _ = crate::mm::munmap(base_addr as *mut u8, map_total_len);
438            }
439        }
440    }
441
442    /// Resizes an existing allocation to `new_size` bytes.
443    ///
444    /// # Safety
445    ///
446    /// `ptr` must be null or a live pointer allocated by this [`MmapHeap`].
447    pub unsafe fn realloc_c(&self, ptr: *mut u8, new_size: usize) -> *mut u8 {
448        if ptr.is_null() {
449            // SAFETY: Delegating to `alloc_c` with 16-byte alignment.
450            return unsafe { self.alloc_c(new_size, 16) };
451        }
452        if new_size == 0 {
453            // SAFETY: Caller guarantees `ptr` is a valid allocation from this heap.
454            unsafe { self.free_c(ptr) };
455            return null_mut();
456        }
457
458        // SAFETY: `ptr` is preceded by a 16-byte metadata header.
459        let hdr = unsafe { ptr.sub(MMAP_HEADER_BYTES).cast::<usize>() };
460        let magic = unsafe { hdr.read() };
461        let old_usable = unsafe { hdr.add(1).read() };
462
463        if (magic & SMALL_MAGIC_MASK) == SMALL_MAGIC_BASE {
464            let cur_class = magic & 0xFF;
465            if let Some(target_class) = mmap_small_class_index(new_size) {
466                if target_class == cur_class {
467                    return ptr;
468                }
469            }
470        } else if magic == LARGE_MAGIC
471            && new_size <= old_usable
472            && new_size > MMAP_MAX_SMALL_PAYLOAD
473        {
474            return ptr;
475        }
476
477        // SAFETY: Allocating replacement buffer of `new_size` bytes.
478        let new_ptr = unsafe { self.alloc_c(new_size, 16) };
479        if new_ptr.is_null() {
480            return null_mut();
481        }
482        let copy_len = if old_usable < new_size {
483            old_usable
484        } else {
485            new_size
486        };
487        // SAFETY: Both `ptr` and `new_ptr` are valid non-overlapping buffers of at least `copy_len` bytes.
488        unsafe {
489            crate::mem::copy_nonoverlapping_bytes(new_ptr, ptr, copy_len);
490            self.free_c(ptr);
491        }
492        new_ptr
493    }
494
495    /// POSIX-compatible aligned allocation writing the result into `*out`.
496    ///
497    /// # Safety
498    ///
499    /// `out` must be a valid non-null writable pointer to `*mut u8`.
500    pub unsafe fn posix_memalign_c(&self, out: *mut *mut u8, align: usize, size: usize) -> i32 {
501        if out.is_null() || align < core::mem::size_of::<usize>() || !align.is_power_of_two() {
502            return crate::errno::EINVAL;
503        }
504        // SAFETY: `align` is a power of two.
505        let ptr = unsafe { self.alloc_c(size, align) };
506        if ptr.is_null() {
507            return crate::errno::ENOMEM;
508        }
509        // SAFETY: `out` was checked for non-null above and caller guarantees writability.
510        unsafe {
511            out.write(ptr);
512        }
513        0
514    }
515}
516
517unsafe impl core::alloc::GlobalAlloc for MmapHeap {
518    unsafe fn alloc(&self, layout: core::alloc::Layout) -> *mut u8 {
519        // SAFETY: Forwarding `GlobalAlloc::alloc` to `alloc_c`.
520        unsafe { self.alloc_c(layout.size(), layout.align()) }
521    }
522
523    unsafe fn alloc_zeroed(&self, layout: core::alloc::Layout) -> *mut u8 {
524        if layout.align() <= 16 {
525            // SAFETY: `calloc_c` returns 16-byte aligned zeroed memory.
526            unsafe { self.calloc_c(1, layout.size()) }
527        } else {
528            // SAFETY: Over-aligned allocation + explicit zeroing when small.
529            let ptr = unsafe { self.alloc_c(layout.size(), layout.align()) };
530            if !ptr.is_null() && layout.size() <= MMAP_MAX_SMALL_PAYLOAD {
531                unsafe {
532                    crate::mem::set_bytes(ptr, 0, layout.size());
533                }
534            }
535            ptr
536        }
537    }
538
539    unsafe fn dealloc(&self, ptr: *mut u8, _layout: core::alloc::Layout) {
540        // SAFETY: `ptr` was allocated by `self`.
541        unsafe { self.free_c(ptr) }
542    }
543
544    unsafe fn realloc(
545        &self,
546        ptr: *mut u8,
547        layout: core::alloc::Layout,
548        new_size: usize,
549    ) -> *mut u8 {
550        if layout.align() <= 16 {
551            // SAFETY: Standard 16-byte aligned reallocation.
552            unsafe { self.realloc_c(ptr, new_size) }
553        } else {
554            // SAFETY: Over-aligned reallocation allocates fresh aligned block, copies, and frees.
555            let new_ptr = unsafe { self.alloc_c(new_size, layout.align()) };
556            if !new_ptr.is_null() {
557                let copy_len = if layout.size() < new_size {
558                    layout.size()
559                } else {
560                    new_size
561                };
562                unsafe {
563                    crate::mem::copy_nonoverlapping_bytes(new_ptr, ptr, copy_len);
564                    self.free_c(ptr);
565                }
566            }
567            new_ptr
568        }
569    }
570}
571
572/// Process-wide global [`MmapHeap`] instance used by `c_malloc`, `c_calloc`, `c_realloc`,
573/// `c_free`, and `c_posix_memalign`.
574pub static GLOBAL_MMAP_HEAP: MmapHeap = MmapHeap::new_cached();
575
576/// Allocates `size` bytes of 16-byte aligned memory from [`GLOBAL_MMAP_HEAP`].
577///
578/// # Safety
579///
580/// Returned memory is uninitialized and must be released with [`c_free`] or [`c_realloc`].
581pub unsafe fn c_malloc(size: usize) -> *mut u8 {
582    // SAFETY: Delegating to `GLOBAL_MMAP_HEAP.alloc_c`.
583    unsafe { GLOBAL_MMAP_HEAP.alloc_c(size, 16) }
584}
585
586/// Allocates and zero-initializes `nobj * size` bytes from [`GLOBAL_MMAP_HEAP`].
587///
588/// # Safety
589///
590/// Returned pointer must be released with [`c_free`] or [`c_realloc`].
591pub unsafe fn c_calloc(nobj: usize, size: usize) -> *mut u8 {
592    // SAFETY: Delegating to `GLOBAL_MMAP_HEAP.calloc_c`.
593    unsafe { GLOBAL_MMAP_HEAP.calloc_c(nobj, size) }
594}
595
596/// Resizes the allocation at `ptr` to `new_size` bytes in [`GLOBAL_MMAP_HEAP`].
597///
598/// # Safety
599///
600/// `ptr` must be null or a pointer previously returned by `c_malloc`, `c_calloc`,
601/// `c_realloc`, or `c_posix_memalign`.
602pub unsafe fn c_realloc(ptr: *mut u8, new_size: usize) -> *mut u8 {
603    // SAFETY: Delegating to `GLOBAL_MMAP_HEAP.realloc_c`.
604    unsafe { GLOBAL_MMAP_HEAP.realloc_c(ptr, new_size) }
605}
606
607/// Frees a pointer previously allocated from [`GLOBAL_MMAP_HEAP`].
608///
609/// # Safety
610///
611/// `ptr` must be null or a live pointer returned by `c_malloc`, `c_calloc`,
612/// `c_realloc`, or `c_posix_memalign`.
613pub unsafe fn c_free(ptr: *mut u8) {
614    // SAFETY: Delegating to `GLOBAL_MMAP_HEAP.free_c`.
615    unsafe { GLOBAL_MMAP_HEAP.free_c(ptr) }
616}
617
618/// Allocates `size` bytes aligned to `align` and writes the pointer to `*out`.
619///
620/// # Safety
621///
622/// `out` must be a valid writable pointer to `*mut u8`.
623pub unsafe fn c_posix_memalign(out: *mut *mut u8, align: usize, size: usize) -> i32 {
624    // SAFETY: Delegating to `GLOBAL_MMAP_HEAP.posix_memalign_c`.
625    unsafe { GLOBAL_MMAP_HEAP.posix_memalign_c(out, align, size) }
626}
627
628#[cfg(test)]
629mod tests {
630    use super::*;
631    use core::sync::atomic::{AtomicUsize, Ordering};
632
633    #[test]
634    fn mmap_heap_small_large_realloc_calloc_and_memalign() {
635        use crate::errno::{ENOMEM, Errno};
636        use crate::raw::{nr, with_mock_syscall};
637
638        #[repr(align(4096))]
639        struct AlignedArena([u8; 256 * 1024]);
640
641        static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
642        static OFFSET: AtomicUsize = AtomicUsize::new(0);
643        static MUNMAP_COUNT: AtomicUsize = AtomicUsize::new(0);
644        static LAST_MUNMAP_LEN: AtomicUsize = AtomicUsize::new(0);
645
646        OFFSET.store(0, Ordering::SeqCst);
647        MUNMAP_COUNT.store(0, Ordering::SeqCst);
648        LAST_MUNMAP_LEN.store(0, Ordering::SeqCst);
649
650        with_mock_syscall(
651            |sysno, args| match sysno {
652                nr::MMAP => {
653                    let len = args[1];
654                    let aligned_len = (len + 4095) & !4095;
655                    let start = OFFSET.fetch_add(aligned_len, Ordering::SeqCst);
656                    if start + aligned_len > 256 * 1024 {
657                        return Err(Errno(ENOMEM));
658                    }
659                    // SAFETY: `start + aligned_len` is within `ARENA`.
660                    let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(start) };
661                    unsafe {
662                        crate::mem::set_bytes(ptr, 0, aligned_len);
663                    }
664                    Ok(ptr as usize)
665                }
666                nr::MUNMAP => {
667                    MUNMAP_COUNT.fetch_add(1, Ordering::SeqCst);
668                    LAST_MUNMAP_LEN.store(args[1], Ordering::SeqCst);
669                    Ok(0)
670                }
671                _ => Err(Errno(crate::errno::ENOSYS)),
672            },
673            || {
674                let p1 = unsafe { c_malloc(24) };
675                assert!(!p1.is_null());
676                assert_eq!((p1 as usize) % 16, 0);
677                unsafe {
678                    crate::mem::set_bytes(p1, 0xAB, 24);
679                    c_free(p1);
680                }
681                let p2 = unsafe { c_malloc(24) };
682                assert_eq!(p1, p2);
683
684                let p_same = unsafe { c_realloc(p2, 40) };
685                assert_eq!(p_same, p2);
686
687                let p_grow = unsafe { c_realloc(p_same, 200) };
688                assert!(!p_grow.is_null());
689                assert_ne!(p_grow, p_same);
690                for i in 0..24 {
691                    assert_eq!(unsafe { *p_grow.add(i) }, 0xAB);
692                }
693                unsafe {
694                    c_free(p_grow);
695                }
696
697                let overflow = unsafe { c_calloc(usize::MAX, 2) };
698                assert!(overflow.is_null());
699                let zeroed = unsafe { c_calloc(8, 16) };
700                assert!(!zeroed.is_null());
701                for i in 0..128 {
702                    assert_eq!(unsafe { *zeroed.add(i) }, 0);
703                }
704                unsafe {
705                    c_free(zeroed);
706                }
707
708                let big = unsafe { c_malloc(8192) };
709                assert!(!big.is_null());
710                assert_eq!((big as usize) % 16, 0);
711                unsafe {
712                    *big = 0x55;
713                    *big.add(8191) = 0x77;
714                    c_free(big);
715                }
716                assert_eq!(MUNMAP_COUNT.load(Ordering::SeqCst), 1);
717                assert!(LAST_MUNMAP_LEN.load(Ordering::SeqCst) >= 8192);
718
719                let mut aligned_out: *mut u8 = null_mut();
720                assert_eq!(
721                    unsafe { c_posix_memalign(&mut aligned_out, 3, 64) },
722                    crate::errno::EINVAL
723                );
724                assert_eq!(
725                    unsafe { c_posix_memalign(&mut aligned_out, 4, 64) },
726                    crate::errno::EINVAL
727                );
728                assert_eq!(unsafe { c_posix_memalign(&mut aligned_out, 4096, 1024) }, 0);
729                assert!(!aligned_out.is_null());
730                assert_eq!((aligned_out as usize) % 4096, 0);
731                unsafe {
732                    c_free(aligned_out);
733                }
734                assert_eq!(MUNMAP_COUNT.load(Ordering::SeqCst), 2);
735            },
736        );
737    }
738
739    #[test]
740    fn mmap_heap_magazine_hit_refill_and_spill_all_classes() {
741        use crate::errno::{ENOMEM, Errno};
742        use crate::raw::{nr, with_mock_syscall};
743
744        #[repr(align(4096))]
745        struct AlignedArena([u8; 512 * 1024]);
746
747        static mut ARENA: AlignedArena = AlignedArena([0; 512 * 1024]);
748        static OFFSET: AtomicUsize = AtomicUsize::new(0);
749
750        OFFSET.store(0, Ordering::SeqCst);
751
752        with_mock_syscall(
753            |sysno, args| match sysno {
754                nr::MMAP => {
755                    let len = args[1];
756                    let aligned_len = (len + 4095) & !4095;
757                    let start = OFFSET.fetch_add(aligned_len, Ordering::SeqCst);
758                    if start + aligned_len > 512 * 1024 {
759                        return Err(Errno(ENOMEM));
760                    }
761                    // SAFETY: `start + aligned_len` is within `ARENA`.
762                    let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(start) };
763                    unsafe {
764                        crate::mem::set_bytes(ptr, 0, aligned_len);
765                    }
766                    Ok(ptr as usize)
767                }
768                nr::MUNMAP => Ok(0),
769                _ => Err(Errno(crate::errno::ENOSYS)),
770            },
771            || {
772                let heap = MmapHeap::new_cached();
773                let uncached = MmapHeap::new();
774                assert!(heap.use_tls_cache);
775                assert!(!uncached.use_tls_cache);
776                let slot_idx = current_magazine_slot_index();
777                unsafe {
778                    *GLOBAL_MAGAZINES[slot_idx].classes.get() = [ClassMagazine::new(); 8];
779                }
780
781                for (class_idx, &total_bytes) in MMAP_CLASS_TOTAL_SIZES.iter().enumerate() {
782                    let payload = total_bytes - MMAP_HEADER_BYTES;
783                    let mut allocated = [null_mut::<u8>(); 20];
784
785                    // First allocation triggers a batch refill of 8 blocks (1 returned + 7 cached).
786                    allocated[0] = unsafe { heap.alloc_c(payload, 16) };
787                    assert!(!allocated[0].is_null());
788                    let mag_len_after_first =
789                        unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
790                    assert_eq!(mag_len_after_first, MAGAZINE_BATCH - 1);
791
792                    // Allocate 19 more blocks across a second and third refill batch.
793                    for (i, slot_ptr) in allocated.iter_mut().enumerate().skip(1) {
794                        let p = unsafe { heap.alloc_c(payload, 16) };
795                        assert!(!p.is_null());
796                        assert_eq!((p as usize) % 16, 0);
797                        unsafe {
798                            crate::mem::set_bytes(
799                                p,
800                                (class_idx as u8).wrapping_add(i as u8),
801                                payload,
802                            );
803                        }
804                        *slot_ptr = p;
805                    }
806
807                    // Drain any remaining cached blocks in this class so we test exact fill and spill.
808                    loop {
809                        let cur_len =
810                            unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
811                        if cur_len == 0 {
812                            break;
813                        }
814                        let extra = unsafe { heap.alloc_c(payload, 16) };
815                        assert!(!extra.is_null());
816                    }
817
818                    // Free 16 blocks to fill the magazine to MAGAZINE_CAP (16).
819                    for &ptr in allocated.iter().take(MAGAZINE_CAP) {
820                        unsafe { heap.free_c(ptr) };
821                    }
822                    let mag_len_full =
823                        unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
824                    assert_eq!(mag_len_full, MAGAZINE_CAP);
825
826                    // Freeing the 17th block spills MAGAZINE_BATCH (8) blocks to MmapHeapInner
827                    // and pushes the new block, leaving `MAGAZINE_CAP - MAGAZINE_BATCH + 1 == 9`.
828                    unsafe { heap.free_c(allocated[MAGAZINE_CAP]) };
829                    let mag_len_after_spill =
830                        unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
831                    assert_eq!(mag_len_after_spill, MAGAZINE_CAP - MAGAZINE_BATCH + 1);
832
833                    // LIFO magazine hit returns the most recently freed block without touching inner.
834                    let hit = unsafe { heap.alloc_c(payload, 16) };
835                    assert_eq!(hit, allocated[MAGAZINE_CAP]);
836
837                    // Verify calloc_c and realloc_c work seamlessly with the magazine cache.
838                    let zeroed = unsafe { heap.calloc_c(1, payload) };
839                    assert!(!zeroed.is_null());
840                    for b in 0..payload {
841                        assert_eq!(unsafe { *zeroed.add(b) }, 0);
842                    }
843                    let shrunk = unsafe { heap.realloc_c(zeroed, 8) };
844                    assert!(!shrunk.is_null());
845                    unsafe {
846                        heap.free_c(shrunk);
847                        heap.free_c(hit);
848                    }
849                    for &ptr in allocated.iter().skip(MAGAZINE_CAP + 1) {
850                        unsafe { heap.free_c(ptr) };
851                    }
852                }
853            },
854        );
855    }
856}