1use core::cell::UnsafeCell;
4use core::ptr::null_mut;
5use core::sync::atomic::{AtomicBool, Ordering};
6
7use crate::sync::FutexMutex;
8
9#[repr(C)]
10struct FreeNode {
11 next: *mut FreeNode,
12}
13
14#[inline]
15const fn align_up(value: usize, align: usize) -> Option<usize> {
16 let mask = align - 1;
17 match value.checked_add(mask) {
18 Some(v) => Some(v & !mask),
19 None => None,
20 }
21}
22
23const MMAP_NUM_CLASSES: usize = 8;
24const _: () = assert!(MMAP_NUM_CLASSES == 8);
25const MMAP_CLASS_TOTAL_SIZES: [usize; MMAP_NUM_CLASSES] = [32, 64, 128, 256, 512, 1024, 2048, 4096];
26const MMAP_HEADER_BYTES: usize = 16;
27const MMAP_LARGE_HEADER_BYTES: usize = 32;
28const MMAP_MAX_SMALL_PAYLOAD: usize = 4080;
29const MMAP_SLAB_BYTES: usize = 16 * crate::mm::PAGE_SIZE;
30
31const SMALL_MAGIC_BASE: usize = 0xC0DE_5A1B_0000_0000;
32const SMALL_MAGIC_MASK: usize = 0xFFFF_FFFF_FFFF_FF00;
33const LARGE_MAGIC: usize = 0xC0DE_1A6E_5A1B_0001;
34
35const MAGAZINE_SLOTS: usize = 64;
36const MAGAZINE_CAP: usize = 16;
37const MAGAZINE_BATCH: usize = 8;
38const _: () = {
39 assert!(MAGAZINE_SLOTS == 64);
40 assert!(MAGAZINE_CAP == 16);
41};
42
43#[derive(Copy, Clone)]
44struct ClassMagazine {
45 len: usize,
46 ptrs: [*mut u8; 16],
47}
48
49impl ClassMagazine {
50 const fn new() -> Self {
51 Self {
52 len: 0,
53 ptrs: [null_mut(); 16],
54 }
55 }
56}
57
58#[repr(align(64))]
59struct ThreadMagazineSlot {
60 in_use: AtomicBool,
61 classes: UnsafeCell<[ClassMagazine; 8]>,
62}
63
64unsafe impl Sync for ThreadMagazineSlot {}
66
67impl ThreadMagazineSlot {
68 const fn new() -> Self {
69 Self {
70 in_use: AtomicBool::new(false),
71 classes: UnsafeCell::new([ClassMagazine::new(); 8]),
72 }
73 }
74}
75
76#[inline]
77fn current_magazine_slot_index() -> usize {
78 let tid = crate::thread::current_thread_id();
79 let mixed = tid ^ (tid >> 6) ^ (tid >> 12) ^ (tid >> 19);
80 (mixed as usize) & (MAGAZINE_SLOTS - 1)
81}
82
83#[inline]
84const fn mmap_small_class_index(size: usize) -> Option<usize> {
85 let total = match size.checked_add(MMAP_HEADER_BYTES) {
86 Some(v) => v,
87 None => return None,
88 };
89 let mut i = 0;
90 while i < MMAP_NUM_CLASSES {
91 if total <= MMAP_CLASS_TOTAL_SIZES[i] {
92 return Some(i);
93 }
94 i += 1;
95 }
96 None
97}
98
99struct MmapHeapInner {
100 free_lists: [*mut FreeNode; 8],
101 slab_ptr: *mut u8,
102 slab_rem: usize,
103}
104
105unsafe impl Send for MmapHeapInner {}
107
108impl MmapHeapInner {
109 const fn new() -> Self {
110 Self {
111 free_lists: [
112 null_mut(),
113 null_mut(),
114 null_mut(),
115 null_mut(),
116 null_mut(),
117 null_mut(),
118 null_mut(),
119 null_mut(),
120 ],
121 slab_ptr: null_mut(),
122 slab_rem: 0,
123 }
124 }
125
126 unsafe fn alloc_small(&mut self, class_idx: usize) -> *mut u8 {
127 let block_bytes = MMAP_CLASS_TOTAL_SIZES[class_idx];
128 let payload_bytes = block_bytes - MMAP_HEADER_BYTES;
129 let head = self.free_lists[class_idx];
130 let block_ptr = if !head.is_null() {
131 unsafe {
133 self.free_lists[class_idx] = (*head).next;
134 }
135 head.cast::<u8>()
136 } else {
137 if self.slab_rem < block_bytes {
138 let slab = match crate::mm::mmap_anonymous(
139 MMAP_SLAB_BYTES,
140 crate::mm::PROT_READ | crate::mm::PROT_WRITE,
141 0,
142 ) {
143 Ok(p) if !p.is_null() => p,
144 _ => return null_mut(),
145 };
146 self.slab_ptr = slab;
147 self.slab_rem = MMAP_SLAB_BYTES;
148 }
149 let p = self.slab_ptr;
150 self.slab_ptr = unsafe { self.slab_ptr.add(block_bytes) };
152 self.slab_rem -= block_bytes;
153 p
154 };
155
156 unsafe {
158 let hdr = block_ptr.cast::<usize>();
159 hdr.write(SMALL_MAGIC_BASE | class_idx);
160 hdr.add(1).write(payload_bytes);
161 block_ptr.add(MMAP_HEADER_BYTES)
162 }
163 }
164
165 unsafe fn free_small(&mut self, block_ptr: *mut u8, class_idx: usize) {
166 let node = block_ptr.cast::<FreeNode>();
167 unsafe {
169 (*node).next = self.free_lists[class_idx];
170 }
171 self.free_lists[class_idx] = node;
172 }
173}
174
175static GLOBAL_MAGAZINES: [ThreadMagazineSlot; 64] = [const { ThreadMagazineSlot::new() }; 64];
176
177pub struct MmapHeap {
181 inner: FutexMutex<MmapHeapInner>,
182 use_tls_cache: bool,
183}
184
185impl Default for MmapHeap {
186 fn default() -> Self {
187 Self::new()
188 }
189}
190
191impl MmapHeap {
192 #[must_use]
194 pub const fn new() -> Self {
195 Self {
196 inner: FutexMutex::new(MmapHeapInner::new()),
197 use_tls_cache: false,
198 }
199 }
200
201 #[must_use]
203 pub const fn new_cached() -> Self {
204 Self {
205 inner: FutexMutex::new(MmapHeapInner::new()),
206 use_tls_cache: true,
207 }
208 }
209
210 #[inline]
211 unsafe fn alloc_small_cached(&self, class_idx: usize) -> *mut u8 {
212 let slot = &GLOBAL_MAGAZINES[current_magazine_slot_index()];
213 if slot
214 .in_use
215 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
216 .is_ok()
217 {
218 let mag = unsafe { &mut (*slot.classes.get())[class_idx] };
220 if mag.len > 0 {
221 mag.len -= 1;
222 let ptr = mag.ptrs[mag.len];
223 slot.in_use.store(false, Ordering::Release);
224 return ptr;
225 }
226
227 let mut inner = self.inner.lock();
228 let first = unsafe { inner.alloc_small(class_idx) };
230 if !first.is_null() {
231 let mut i = 1;
232 while i < MAGAZINE_BATCH && mag.len < MAGAZINE_CAP {
233 let extra = unsafe { inner.alloc_small(class_idx) };
235 if extra.is_null() {
236 break;
237 }
238 mag.ptrs[mag.len] = extra;
239 mag.len += 1;
240 i += 1;
241 }
242 }
243 drop(inner);
244 slot.in_use.store(false, Ordering::Release);
245 return first;
246 }
247
248 let mut inner = self.inner.lock();
249 unsafe { inner.alloc_small(class_idx) }
251 }
252
253 #[inline]
254 unsafe fn free_small_cached(&self, ptr: *mut u8, block_ptr: *mut u8, class_idx: usize) {
255 let slot = &GLOBAL_MAGAZINES[current_magazine_slot_index()];
256 if slot
257 .in_use
258 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
259 .is_ok()
260 {
261 let mag = unsafe { &mut (*slot.classes.get())[class_idx] };
263 if mag.len < MAGAZINE_CAP {
264 mag.ptrs[mag.len] = ptr;
265 mag.len += 1;
266 slot.in_use.store(false, Ordering::Release);
267 return;
268 }
269
270 let mut inner = self.inner.lock();
271 let mut spilled = 0;
272 while spilled < MAGAZINE_BATCH && mag.len > 0 {
273 mag.len -= 1;
274 let victim_ptr = mag.ptrs[mag.len];
275 let victim_block = unsafe { victim_ptr.sub(MMAP_HEADER_BYTES) };
277 unsafe { inner.free_small(victim_block, class_idx) };
278 spilled += 1;
279 }
280 drop(inner);
281 mag.ptrs[mag.len] = ptr;
282 mag.len += 1;
283 slot.in_use.store(false, Ordering::Release);
284 return;
285 }
286
287 let mut inner = self.inner.lock();
288 unsafe { inner.free_small(block_ptr, class_idx) };
290 }
291
292 pub unsafe fn alloc_c(&self, size: usize, align: usize) -> *mut u8 {
299 let eff_align = if align < 16 { 16 } else { align };
300 if !eff_align.is_power_of_two() {
301 return null_mut();
302 }
303 let req_size = if size == 0 { 1 } else { size };
304
305 if eff_align == 16 && req_size <= MMAP_MAX_SMALL_PAYLOAD {
306 if let Some(class_idx) = mmap_small_class_index(req_size) {
307 if self.use_tls_cache {
308 return unsafe { self.alloc_small_cached(class_idx) };
310 }
311 let mut inner = self.inner.lock();
312 return unsafe { inner.alloc_small(class_idx) };
314 }
315 }
316
317 unsafe { Self::alloc_large(req_size, eff_align) }
319 }
320
321 unsafe fn alloc_large(size: usize, align: usize) -> *mut u8 {
322 let extra = match align.checked_add(MMAP_LARGE_HEADER_BYTES) {
323 Some(v) => v,
324 None => return null_mut(),
325 };
326 let raw_needed = match size.checked_add(extra) {
327 Some(v) => v,
328 None => return null_mut(),
329 };
330 let map_total_len = match align_up(raw_needed, crate::mm::PAGE_SIZE) {
331 Some(v) => v,
332 None => return null_mut(),
333 };
334
335 let map_base = match crate::mm::mmap_anonymous(
336 map_total_len,
337 crate::mm::PROT_READ | crate::mm::PROT_WRITE,
338 0,
339 ) {
340 Ok(p) if !p.is_null() => p,
341 _ => return null_mut(),
342 };
343
344 let base_addr = map_base as usize;
345 let min_user_addr = match base_addr.checked_add(MMAP_LARGE_HEADER_BYTES) {
346 Some(v) => v,
347 None => {
348 let _ = unsafe { crate::mm::munmap(map_base, map_total_len) };
350 return null_mut();
351 }
352 };
353 let user_addr = match align_up(min_user_addr, align) {
354 Some(v) => v,
355 None => {
356 let _ = unsafe { crate::mm::munmap(map_base, map_total_len) };
358 return null_mut();
359 }
360 };
361 let offset = user_addr - base_addr;
362 let usable_size = map_total_len.saturating_sub(offset);
363 unsafe {
365 let user_ptr = map_base.add(offset);
366 let hdr = user_ptr.sub(MMAP_LARGE_HEADER_BYTES).cast::<usize>();
367 hdr.write(base_addr);
368 hdr.add(1).write(map_total_len);
369 hdr.add(2).write(LARGE_MAGIC);
370 hdr.add(3).write(usable_size);
371 user_ptr
372 }
373 }
374
375 pub unsafe fn calloc_c(&self, nobj: usize, size: usize) -> *mut u8 {
381 let total = match nobj.checked_mul(size) {
382 Some(v) => v,
383 None => return null_mut(),
384 };
385 let req_size = if total == 0 { 1 } else { total };
386 if req_size <= MMAP_MAX_SMALL_PAYLOAD {
387 let ptr = unsafe { self.alloc_c(req_size, 16) };
389 if !ptr.is_null() {
390 unsafe {
392 crate::mem::set_bytes(ptr, 0, req_size);
393 }
394 }
395 return ptr;
396 }
397 unsafe { Self::alloc_large(req_size, 16) }
399 }
400
401 pub unsafe fn free_c(&self, ptr: *mut u8) {
408 if ptr.is_null() {
409 return;
410 }
411 let hdr = unsafe { ptr.sub(MMAP_HEADER_BYTES).cast::<usize>() };
413 let magic = unsafe { hdr.read() };
414
415 if (magic & SMALL_MAGIC_MASK) == SMALL_MAGIC_BASE {
416 let class_idx = magic & 0xFF;
417 if class_idx < MMAP_NUM_CLASSES {
418 let block_ptr = hdr.cast::<u8>();
419 if self.use_tls_cache {
420 unsafe { self.free_small_cached(ptr, block_ptr, class_idx) };
422 return;
423 }
424 let mut inner = self.inner.lock();
425 unsafe { inner.free_small(block_ptr, class_idx) };
427 }
428 return;
429 }
430
431 if magic == LARGE_MAGIC {
432 unsafe {
434 let large_hdr = ptr.sub(MMAP_LARGE_HEADER_BYTES).cast::<usize>();
435 let base_addr = large_hdr.read();
436 let map_total_len = large_hdr.add(1).read();
437 let _ = crate::mm::munmap(base_addr as *mut u8, map_total_len);
438 }
439 }
440 }
441
442 pub unsafe fn realloc_c(&self, ptr: *mut u8, new_size: usize) -> *mut u8 {
448 if ptr.is_null() {
449 return unsafe { self.alloc_c(new_size, 16) };
451 }
452 if new_size == 0 {
453 unsafe { self.free_c(ptr) };
455 return null_mut();
456 }
457
458 let hdr = unsafe { ptr.sub(MMAP_HEADER_BYTES).cast::<usize>() };
460 let magic = unsafe { hdr.read() };
461 let old_usable = unsafe { hdr.add(1).read() };
462
463 if (magic & SMALL_MAGIC_MASK) == SMALL_MAGIC_BASE {
464 let cur_class = magic & 0xFF;
465 if let Some(target_class) = mmap_small_class_index(new_size) {
466 if target_class == cur_class {
467 return ptr;
468 }
469 }
470 } else if magic == LARGE_MAGIC
471 && new_size <= old_usable
472 && new_size > MMAP_MAX_SMALL_PAYLOAD
473 {
474 return ptr;
475 }
476
477 let new_ptr = unsafe { self.alloc_c(new_size, 16) };
479 if new_ptr.is_null() {
480 return null_mut();
481 }
482 let copy_len = if old_usable < new_size {
483 old_usable
484 } else {
485 new_size
486 };
487 unsafe {
489 crate::mem::copy_nonoverlapping_bytes(new_ptr, ptr, copy_len);
490 self.free_c(ptr);
491 }
492 new_ptr
493 }
494
495 pub unsafe fn posix_memalign_c(&self, out: *mut *mut u8, align: usize, size: usize) -> i32 {
501 if out.is_null() || align < core::mem::size_of::<usize>() || !align.is_power_of_two() {
502 return crate::errno::EINVAL;
503 }
504 let ptr = unsafe { self.alloc_c(size, align) };
506 if ptr.is_null() {
507 return crate::errno::ENOMEM;
508 }
509 unsafe {
511 out.write(ptr);
512 }
513 0
514 }
515}
516
517unsafe impl core::alloc::GlobalAlloc for MmapHeap {
518 unsafe fn alloc(&self, layout: core::alloc::Layout) -> *mut u8 {
519 unsafe { self.alloc_c(layout.size(), layout.align()) }
521 }
522
523 unsafe fn alloc_zeroed(&self, layout: core::alloc::Layout) -> *mut u8 {
524 if layout.align() <= 16 {
525 unsafe { self.calloc_c(1, layout.size()) }
527 } else {
528 let ptr = unsafe { self.alloc_c(layout.size(), layout.align()) };
530 if !ptr.is_null() && layout.size() <= MMAP_MAX_SMALL_PAYLOAD {
531 unsafe {
532 crate::mem::set_bytes(ptr, 0, layout.size());
533 }
534 }
535 ptr
536 }
537 }
538
539 unsafe fn dealloc(&self, ptr: *mut u8, _layout: core::alloc::Layout) {
540 unsafe { self.free_c(ptr) }
542 }
543
544 unsafe fn realloc(
545 &self,
546 ptr: *mut u8,
547 layout: core::alloc::Layout,
548 new_size: usize,
549 ) -> *mut u8 {
550 if layout.align() <= 16 {
551 unsafe { self.realloc_c(ptr, new_size) }
553 } else {
554 let new_ptr = unsafe { self.alloc_c(new_size, layout.align()) };
556 if !new_ptr.is_null() {
557 let copy_len = if layout.size() < new_size {
558 layout.size()
559 } else {
560 new_size
561 };
562 unsafe {
563 crate::mem::copy_nonoverlapping_bytes(new_ptr, ptr, copy_len);
564 self.free_c(ptr);
565 }
566 }
567 new_ptr
568 }
569 }
570}
571
572pub static GLOBAL_MMAP_HEAP: MmapHeap = MmapHeap::new_cached();
575
576pub unsafe fn c_malloc(size: usize) -> *mut u8 {
582 unsafe { GLOBAL_MMAP_HEAP.alloc_c(size, 16) }
584}
585
586pub unsafe fn c_calloc(nobj: usize, size: usize) -> *mut u8 {
592 unsafe { GLOBAL_MMAP_HEAP.calloc_c(nobj, size) }
594}
595
596pub unsafe fn c_realloc(ptr: *mut u8, new_size: usize) -> *mut u8 {
603 unsafe { GLOBAL_MMAP_HEAP.realloc_c(ptr, new_size) }
605}
606
607pub unsafe fn c_free(ptr: *mut u8) {
614 unsafe { GLOBAL_MMAP_HEAP.free_c(ptr) }
616}
617
618pub unsafe fn c_posix_memalign(out: *mut *mut u8, align: usize, size: usize) -> i32 {
624 unsafe { GLOBAL_MMAP_HEAP.posix_memalign_c(out, align, size) }
626}
627
628#[cfg(test)]
629mod tests {
630 use super::*;
631 use core::sync::atomic::{AtomicUsize, Ordering};
632
633 #[test]
634 fn mmap_heap_small_large_realloc_calloc_and_memalign() {
635 use crate::errno::{ENOMEM, Errno};
636 use crate::raw::{nr, with_mock_syscall};
637
638 #[repr(align(4096))]
639 struct AlignedArena([u8; 256 * 1024]);
640
641 static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
642 static OFFSET: AtomicUsize = AtomicUsize::new(0);
643 static MUNMAP_COUNT: AtomicUsize = AtomicUsize::new(0);
644 static LAST_MUNMAP_LEN: AtomicUsize = AtomicUsize::new(0);
645
646 OFFSET.store(0, Ordering::SeqCst);
647 MUNMAP_COUNT.store(0, Ordering::SeqCst);
648 LAST_MUNMAP_LEN.store(0, Ordering::SeqCst);
649
650 with_mock_syscall(
651 |sysno, args| match sysno {
652 nr::MMAP => {
653 let len = args[1];
654 let aligned_len = (len + 4095) & !4095;
655 let start = OFFSET.fetch_add(aligned_len, Ordering::SeqCst);
656 if start + aligned_len > 256 * 1024 {
657 return Err(Errno(ENOMEM));
658 }
659 let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(start) };
661 unsafe {
662 crate::mem::set_bytes(ptr, 0, aligned_len);
663 }
664 Ok(ptr as usize)
665 }
666 nr::MUNMAP => {
667 MUNMAP_COUNT.fetch_add(1, Ordering::SeqCst);
668 LAST_MUNMAP_LEN.store(args[1], Ordering::SeqCst);
669 Ok(0)
670 }
671 _ => Err(Errno(crate::errno::ENOSYS)),
672 },
673 || {
674 let p1 = unsafe { c_malloc(24) };
675 assert!(!p1.is_null());
676 assert_eq!((p1 as usize) % 16, 0);
677 unsafe {
678 crate::mem::set_bytes(p1, 0xAB, 24);
679 c_free(p1);
680 }
681 let p2 = unsafe { c_malloc(24) };
682 assert_eq!(p1, p2);
683
684 let p_same = unsafe { c_realloc(p2, 40) };
685 assert_eq!(p_same, p2);
686
687 let p_grow = unsafe { c_realloc(p_same, 200) };
688 assert!(!p_grow.is_null());
689 assert_ne!(p_grow, p_same);
690 for i in 0..24 {
691 assert_eq!(unsafe { *p_grow.add(i) }, 0xAB);
692 }
693 unsafe {
694 c_free(p_grow);
695 }
696
697 let overflow = unsafe { c_calloc(usize::MAX, 2) };
698 assert!(overflow.is_null());
699 let zeroed = unsafe { c_calloc(8, 16) };
700 assert!(!zeroed.is_null());
701 for i in 0..128 {
702 assert_eq!(unsafe { *zeroed.add(i) }, 0);
703 }
704 unsafe {
705 c_free(zeroed);
706 }
707
708 let big = unsafe { c_malloc(8192) };
709 assert!(!big.is_null());
710 assert_eq!((big as usize) % 16, 0);
711 unsafe {
712 *big = 0x55;
713 *big.add(8191) = 0x77;
714 c_free(big);
715 }
716 assert_eq!(MUNMAP_COUNT.load(Ordering::SeqCst), 1);
717 assert!(LAST_MUNMAP_LEN.load(Ordering::SeqCst) >= 8192);
718
719 let mut aligned_out: *mut u8 = null_mut();
720 assert_eq!(
721 unsafe { c_posix_memalign(&mut aligned_out, 3, 64) },
722 crate::errno::EINVAL
723 );
724 assert_eq!(
725 unsafe { c_posix_memalign(&mut aligned_out, 4, 64) },
726 crate::errno::EINVAL
727 );
728 assert_eq!(unsafe { c_posix_memalign(&mut aligned_out, 4096, 1024) }, 0);
729 assert!(!aligned_out.is_null());
730 assert_eq!((aligned_out as usize) % 4096, 0);
731 unsafe {
732 c_free(aligned_out);
733 }
734 assert_eq!(MUNMAP_COUNT.load(Ordering::SeqCst), 2);
735 },
736 );
737 }
738
739 #[test]
740 fn mmap_heap_magazine_hit_refill_and_spill_all_classes() {
741 use crate::errno::{ENOMEM, Errno};
742 use crate::raw::{nr, with_mock_syscall};
743
744 #[repr(align(4096))]
745 struct AlignedArena([u8; 512 * 1024]);
746
747 static mut ARENA: AlignedArena = AlignedArena([0; 512 * 1024]);
748 static OFFSET: AtomicUsize = AtomicUsize::new(0);
749
750 OFFSET.store(0, Ordering::SeqCst);
751
752 with_mock_syscall(
753 |sysno, args| match sysno {
754 nr::MMAP => {
755 let len = args[1];
756 let aligned_len = (len + 4095) & !4095;
757 let start = OFFSET.fetch_add(aligned_len, Ordering::SeqCst);
758 if start + aligned_len > 512 * 1024 {
759 return Err(Errno(ENOMEM));
760 }
761 let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(start) };
763 unsafe {
764 crate::mem::set_bytes(ptr, 0, aligned_len);
765 }
766 Ok(ptr as usize)
767 }
768 nr::MUNMAP => Ok(0),
769 _ => Err(Errno(crate::errno::ENOSYS)),
770 },
771 || {
772 let heap = MmapHeap::new_cached();
773 let uncached = MmapHeap::new();
774 assert!(heap.use_tls_cache);
775 assert!(!uncached.use_tls_cache);
776 let slot_idx = current_magazine_slot_index();
777 unsafe {
778 *GLOBAL_MAGAZINES[slot_idx].classes.get() = [ClassMagazine::new(); 8];
779 }
780
781 for (class_idx, &total_bytes) in MMAP_CLASS_TOTAL_SIZES.iter().enumerate() {
782 let payload = total_bytes - MMAP_HEADER_BYTES;
783 let mut allocated = [null_mut::<u8>(); 20];
784
785 allocated[0] = unsafe { heap.alloc_c(payload, 16) };
787 assert!(!allocated[0].is_null());
788 let mag_len_after_first =
789 unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
790 assert_eq!(mag_len_after_first, MAGAZINE_BATCH - 1);
791
792 for (i, slot_ptr) in allocated.iter_mut().enumerate().skip(1) {
794 let p = unsafe { heap.alloc_c(payload, 16) };
795 assert!(!p.is_null());
796 assert_eq!((p as usize) % 16, 0);
797 unsafe {
798 crate::mem::set_bytes(
799 p,
800 (class_idx as u8).wrapping_add(i as u8),
801 payload,
802 );
803 }
804 *slot_ptr = p;
805 }
806
807 loop {
809 let cur_len =
810 unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
811 if cur_len == 0 {
812 break;
813 }
814 let extra = unsafe { heap.alloc_c(payload, 16) };
815 assert!(!extra.is_null());
816 }
817
818 for &ptr in allocated.iter().take(MAGAZINE_CAP) {
820 unsafe { heap.free_c(ptr) };
821 }
822 let mag_len_full =
823 unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
824 assert_eq!(mag_len_full, MAGAZINE_CAP);
825
826 unsafe { heap.free_c(allocated[MAGAZINE_CAP]) };
829 let mag_len_after_spill =
830 unsafe { (*GLOBAL_MAGAZINES[slot_idx].classes.get())[class_idx].len };
831 assert_eq!(mag_len_after_spill, MAGAZINE_CAP - MAGAZINE_BATCH + 1);
832
833 let hit = unsafe { heap.alloc_c(payload, 16) };
835 assert_eq!(hit, allocated[MAGAZINE_CAP]);
836
837 let zeroed = unsafe { heap.calloc_c(1, payload) };
839 assert!(!zeroed.is_null());
840 for b in 0..payload {
841 assert_eq!(unsafe { *zeroed.add(b) }, 0);
842 }
843 let shrunk = unsafe { heap.realloc_c(zeroed, 8) };
844 assert!(!shrunk.is_null());
845 unsafe {
846 heap.free_c(shrunk);
847 heap.free_c(hit);
848 }
849 for &ptr in allocated.iter().skip(MAGAZINE_CAP + 1) {
850 unsafe { heap.free_c(ptr) };
851 }
852 }
853 },
854 );
855 }
856}