use core::ptr;
use core::sync::atomic::Ordering;
use crate::heap::Heap;
use crate::init;
use crate::page::{Block, Page, pflags, remote_free};
use crate::segment::{self, Segment, SegmentKind, page_of, segment_of};
use crate::types::{BIN_HUGE, SMALL_SIZE_MAX};
unsafe fn unalign(pg: *mut Page, p: *mut u8) -> *mut u8 {
unsafe {
if (*pg).flags.load(Ordering::Relaxed) & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK) == 0 {
return p;
}
let area = (*pg).area;
debug_assert!(
{
let seg = segment_of(p);
area == segment::page_area(seg, segment::page_index(seg, pg))
},
"unalign: Page::area disagrees with page_area(page_index(..))"
);
if (*pg).flags.load(Ordering::Relaxed) & pflags::SINGLE_BLOCK != 0 {
return area;
}
let off = p.addr() - area.addr();
let bsize = (*pg).block_size;
area.add(crate::bins::div_by_block_size(off, bsize) * bsize)
}
}
#[inline(always)]
unsafe fn free_local_owned(seg: *mut Segment, pg: *mut Page, block: *mut u8) {
unsafe {
let xh = (*pg).xheap.load(core::sync::atomic::Ordering::Acquire);
if xh == 0 {
return free_local_no_xheap(seg, pg, block);
}
(*(*init::box_of_xheap(xh)).heap.get()).free_local_at(seg, pg, block);
}
}
#[cold]
#[inline(never)]
unsafe fn free_local_no_xheap(seg: *mut Segment, pg: *mut Page, block: *mut u8) {
unsafe { (*owner_heap(pg)).free_local_at(seg, pg, block) };
}
#[inline]
unsafe fn owner_heap(pg: *mut Page) -> *mut Heap {
unsafe {
let xh = (*pg).xheap.load(core::sync::atomic::Ordering::Acquire);
if xh != 0 {
(*init::box_of_xheap(xh)).heap.get()
} else {
let h = my_heap();
debug_assert!(!h.is_null(), "owner_heap fallback on a heapless thread");
h
}
}
}
#[inline]
fn my_heap() -> *mut Heap {
let hb = init::heap_box();
if hb.is_null() {
return core::ptr::null_mut();
}
unsafe { (*hb).heap.get() }
}
#[inline(always)]
fn debug_foreign_pointer_guard(p: *mut u8) {
#[cfg(any(debug_assertions, feature = "debug_checks"))]
{
assert!(
crate::segment_map::contains(p),
"rusty_alloc: free() called on a pointer this allocator never returned \
({p:p} is not in any registered segment window). A foreign pointer here \
reads metadata from memory we do not own — see UNSAFE.md and R-001."
);
}
#[cfg(not(any(debug_assertions, feature = "debug_checks")))]
{
let _ = p;
}
}
#[inline]
fn stat_realloc(in_place: bool) {
#[cfg(debug_assertions)]
{
let h = my_heap();
if !h.is_null() {
unsafe {
if in_place {
(*h).stats.realloc_in_place += 1;
} else {
(*h).stats.realloc_moved += 1;
}
}
}
}
#[cfg(not(debug_assertions))]
{
let _ = in_place;
}
}
#[inline]
pub fn malloc(size: usize) -> *mut u8 {
malloc_impl::<true, true>(size)
}
#[inline(always)]
fn malloc_impl<const SIZE_FIRST: bool, const DRY: bool>(size: usize) -> *mut u8 {
let hb = init::heap_box_fast();
unsafe {
if size <= SMALL_SIZE_MAX {
let w = crate::types::wsize_from_size(size);
let h = (*hb).heap.get();
let p = (*h).direct[w];
let b = crate::page::page_pop(p);
if !b.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
return b;
}
if DRY {
return if SIZE_FIRST {
malloc_dry(size, hb, p)
} else {
malloc_dry_hb_first(hb, size, p)
};
}
}
if SIZE_FIRST {
malloc_slow(size, hb)
} else {
malloc_slow_hb_first(hb, size)
}
}
}
#[cold]
#[inline(never)]
unsafe fn malloc_dry(size: usize, hb: *mut init::HeapBox, pg: *mut Page) -> *mut u8 {
unsafe {
if let Some(b) = swap_trip(hb, pg) {
return b;
}
malloc_slow(size, hb)
}
}
#[cold]
#[inline(never)]
unsafe fn malloc_dry_hb_first(hb: *mut init::HeapBox, size: usize, pg: *mut Page) -> *mut u8 {
unsafe {
if let Some(b) = swap_trip(hb, pg) {
return b;
}
malloc_slow_hb_first(hb, size)
}
}
#[inline(always)]
unsafe fn swap_trip(hb: *mut init::HeapBox, pg: *mut Page) -> Option<*mut u8> {
unsafe {
let lf = (*pg).local_free;
if lf.is_null() {
return None;
}
let h = (*hb).heap.get();
let quiet = (crate::ONE_THREAD || (*hb).delayed.head.load(Ordering::Acquire) == 0)
&& !(crate::GUARD_PAGES && (*h).guarded_rate != 0)
&& !crate::options::deferred_registered();
if !quiet {
return None;
}
let left = (*h).generic_countdown.wrapping_sub(1);
(*h).generic_countdown = left;
if left == usize::MAX {
(*h).generic_countdown = 0;
return None;
}
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
Some(crate::page::page_swap_pop(pg, lf))
}
}
#[inline]
pub fn malloc_or(size: usize, on_oom: fn(usize) -> *mut u8) -> *mut u8 {
let hb = init::heap_box_fast();
unsafe {
if size <= SMALL_SIZE_MAX {
let w = crate::types::wsize_from_size(size);
let h = (*hb).heap.get();
let p = (*h).direct[w];
let b = crate::page::page_pop(p);
if !b.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
return b;
}
}
malloc_or_slow(size, hb, on_oom)
}
}
pub trait OnOom {
fn on_oom(size: usize) -> *mut u8;
}
#[inline]
pub fn malloc_or_with<H: OnOom>(size: usize) -> *mut u8 {
let hb = init::heap_box_fast();
unsafe {
if size <= SMALL_SIZE_MAX {
let w = crate::types::wsize_from_size(size);
let h = (*hb).heap.get();
let p = (*h).direct[w];
let b = crate::page::page_pop(p);
if !b.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
return b;
}
return malloc_or_dry_with::<H>(size, hb, p);
}
malloc_or_slow_with::<H>(size, hb)
}
}
#[cold]
#[inline(never)]
unsafe fn malloc_or_dry_with<H: OnOom>(
size: usize,
hb: *mut init::HeapBox,
pg: *mut Page,
) -> *mut u8 {
unsafe {
if let Some(b) = swap_trip(hb, pg) {
return b;
}
malloc_or_slow_with::<H>(size, hb)
}
}
#[cold]
#[inline(never)]
unsafe fn malloc_or_slow_with<H: OnOom>(size: usize, hb: *mut init::HeapBox) -> *mut u8 {
let p = unsafe { malloc_slow(size, hb) };
if p.is_null() { H::on_oom(size) } else { p }
}
#[cold]
#[inline(never)]
unsafe fn malloc_or_slow(
size: usize,
hb: *mut init::HeapBox,
on_oom: fn(usize) -> *mut u8,
) -> *mut u8 {
let p = unsafe { malloc_slow(size, hb) };
if p.is_null() { on_oom(size) } else { p }
}
#[cold]
#[inline(never)]
unsafe fn malloc_slow(size: usize, hb: *mut init::HeapBox) -> *mut u8 {
unsafe { malloc_slow_body(size, hb) }
}
#[cold]
#[inline(never)]
unsafe fn malloc_slow_hb_first(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe { malloc_slow_body(size, hb) }
}
#[inline(always)]
unsafe fn malloc_slow_body(size: usize, hb: *mut init::HeapBox) -> *mut u8 {
if SENTINEL_BY_FLAG {
unsafe {
let h = (*hb).heap.get();
if (*h).deferred_due {
return malloc_slow_rare(size, hb);
}
return (*h).malloc_generic(size).0;
}
}
if hb != init::empty_heap_box_ptr() {
unsafe {
let h = (*hb).heap.get();
if (*h).deferred_due {
return malloc_slow_deferred(h, size);
}
return (*h).malloc_generic(size).0;
}
}
malloc_first(size)
}
const SENTINEL_BY_FLAG: bool = true;
#[cold]
#[inline(never)]
unsafe fn malloc_slow_rare(size: usize, hb: *mut init::HeapBox) -> *mut u8 {
if hb == init::empty_heap_box_ptr() {
return malloc_first(size);
}
unsafe { malloc_slow_deferred((*hb).heap.get(), size) }
}
#[cold]
#[inline(never)]
unsafe fn malloc_slow_deferred(h: *mut crate::heap::Heap, size: usize) -> *mut u8 {
unsafe {
fire_deferred_if_due(h);
(*h).malloc_generic(size).0
}
}
#[inline(always)]
unsafe fn fire_deferred_if_due(h: *mut crate::heap::Heap) {
unsafe {
if (*h).deferred_due {
(*h).deferred_due = false;
fire_deferred_now();
}
}
}
#[cold]
#[inline(never)]
fn fire_deferred_now() {
crate::options::deferred_free(false);
}
#[cold]
#[inline(never)]
fn malloc_first(size: usize) -> *mut u8 {
#[cfg(ra_shared_light_heap)]
if let Some(p) = crate::shared_light::try_alloc(size, false) {
return p;
}
let hb = init::ensure_heap(init::empty_heap_box_ptr());
if hb.is_null() {
return ptr::null_mut();
}
unsafe { (*(*hb).heap.get()).malloc_generic(size).0 }
}
#[inline]
pub fn zalloc(size: usize) -> *mut u8 {
let hb = init::heap_box_fast();
unsafe {
if size <= SMALL_SIZE_MAX {
let w = crate::types::wsize_from_size(size);
let h = (*hb).heap.get();
let p = (*h).direct[w];
let b = crate::page::page_pop(p);
if !b.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
if (*p).free_is_zero {
b.cast::<usize>().write(0);
} else {
ptr::write_bytes(b, 0, (*p).block_size);
}
return b;
}
return zalloc_dry(hb, size, p);
}
zalloc_slow(hb, size)
}
}
#[cold]
#[inline(never)]
unsafe fn zalloc_dry(hb: *mut init::HeapBox, size: usize, pg: *mut Page) -> *mut u8 {
unsafe {
if let Some(b) = swap_trip(hb, pg) {
ptr::write_bytes(b, 0, (*pg).block_size);
return b;
}
zalloc_slow(hb, size)
}
}
#[cold]
#[inline(never)]
unsafe fn zalloc_slow(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe {
let h = (*hb).heap.get();
if (*h).deferred_due {
return zalloc_slow_rare(hb, size);
}
(*h).zalloc(size)
}
}
#[cold]
#[inline(never)]
unsafe fn zalloc_slow_rare(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
if hb == init::empty_heap_box_ptr() {
return zalloc_first(size);
}
unsafe {
let h = (*hb).heap.get();
fire_deferred_if_due(h);
(*h).zalloc(size)
}
}
#[cold]
#[inline(never)]
fn zalloc_first(size: usize) -> *mut u8 {
#[cfg(ra_shared_light_heap)]
if let Some(p) = crate::shared_light::try_alloc(size, true) {
return p;
}
let h = my_heap();
if h.is_null() {
return ptr::null_mut(); }
unsafe { (*h).zalloc(size) }
}
unsafe fn zero_block(p: *mut u8, is_zero: bool) {
if is_zero {
unsafe { p.cast::<usize>().write(0) };
} else {
unsafe { core::ptr::write_bytes(p, 0, usable_size(p)) };
}
}
#[inline]
pub fn calloc(count: usize, size: usize) -> *mut u8 {
match count.checked_mul(size) {
Some(total) => zalloc(total),
None => ptr::null_mut(),
}
}
pub fn mallocn(count: usize, size: usize) -> *mut u8 {
match count.checked_mul(size) {
Some(total) => malloc(total),
None => ptr::null_mut(),
}
}
pub fn malloc_small(size: usize) -> *mut u8 {
malloc(size)
}
pub fn zalloc_small(size: usize) -> *mut u8 {
zalloc(size)
}
pub fn malloc_aligned(size: usize, align: usize) -> *mut u8 {
malloc_aligned_at(size, align, 0)
}
pub fn malloc_aligned_c(align: usize, size: usize) -> *mut u8 {
aligned_at_impl::<false>(size, align, 0)
}
pub fn malloc_aligned_at(size: usize, align: usize, offset: usize) -> *mut u8 {
aligned_at_impl::<false>(size, align, offset)
}
#[inline]
pub unsafe fn malloc_aligned_pow2(size: usize, align: usize) -> *mut u8 {
debug_assert!(align.is_power_of_two(), "malloc_aligned_pow2: {align}");
aligned_at_impl::<true>(size, align, 0)
}
#[inline(always)]
fn aligned_at_impl<const POW2: bool>(size: usize, align: usize, offset: usize) -> *mut u8 {
let hb = init::heap_box_fast();
unsafe {
let mask = align.wrapping_sub(1);
if offset == 0
&& size <= SMALL_SIZE_MAX
&& mask < crate::types::SEGMENT_SIZE / 2
&& (POW2 || align & mask == 0)
{
let h = (*hb).heap.get();
let w = crate::types::wsize_from_size(size);
let p = (*h).direct[w];
let b = (*p).free;
if !b.is_null() && b.addr() & mask == 0 {
let blk = crate::page::page_pop(p);
if !blk.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
return blk;
}
}
}
malloc_aligned_slow(size, align, offset, hb)
}
}
#[cold]
#[inline(never)]
unsafe fn malloc_aligned_slow(
size: usize,
align: usize,
offset: usize,
hb: *mut init::HeapBox,
) -> *mut u8 {
if hb != init::empty_heap_box_ptr() {
unsafe {
let mask = align.wrapping_sub(1);
if offset == 0
&& size > SMALL_SIZE_MAX
&& size <= crate::types::MEDIUM_OBJ_SIZE_MAX
&& mask < crate::types::SEGMENT_SIZE / 2
&& align & mask == 0
{
let h = (*hb).heap.get();
let p = (*h).pages[crate::bins::bin(size)].first;
if !p.is_null() {
let b = (*p).free;
if !b.is_null() && b.addr() & mask == 0 {
let blk = crate::page::page_pop(p);
if !blk.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
return blk;
}
}
}
}
}
return unsafe {
(*(*hb).heap.get())
.malloc_aligned_at_slow(size, align, offset)
.0
};
}
malloc_aligned_first(size, align, offset)
}
#[cold]
#[inline(never)]
fn malloc_aligned_first(size: usize, align: usize, offset: usize) -> *mut u8 {
let h = my_heap();
if h.is_null() {
return ptr::null_mut(); }
unsafe {
fire_deferred_if_due(h);
(*h).malloc_aligned_at(size, align, offset).0
}
}
pub fn zalloc_aligned(size: usize, align: usize) -> *mut u8 {
zalloc_aligned_at(size, align, 0)
}
pub fn zalloc_aligned_at(size: usize, align: usize, offset: usize) -> *mut u8 {
let hb = init::heap_box_fast();
if hb == init::empty_heap_box_ptr() {
return zalloc_aligned_first(size, align, offset);
}
unsafe {
let h = (*hb).heap.get();
fire_deferred_if_due(h);
let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset);
if !p.is_null() {
zero_block(p, is_zero);
}
p
}
}
#[cold]
#[inline(never)]
fn zalloc_aligned_first(size: usize, align: usize, offset: usize) -> *mut u8 {
let h = my_heap();
if h.is_null() {
return ptr::null_mut(); }
unsafe {
fire_deferred_if_due(h);
let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset);
if !p.is_null() {
zero_block(p, is_zero);
}
p
}
}
pub fn calloc_aligned(count: usize, size: usize, align: usize) -> *mut u8 {
calloc_aligned_at(count, size, align, 0)
}
pub fn calloc_aligned_at(count: usize, size: usize, align: usize, offset: usize) -> *mut u8 {
match count.checked_mul(size) {
Some(total) => zalloc_aligned_at(total, align, offset),
None => ptr::null_mut(),
}
}
pub unsafe fn realloc_aligned(p: *mut u8, newsize: usize, align: usize) -> *mut u8 {
unsafe { realloc_aligned_at(p, newsize, align, 0) }
}
pub unsafe fn realloc_aligned_at(
p: *mut u8,
newsize: usize,
align: usize,
offset: usize,
) -> *mut u8 {
if p.is_null() {
return malloc_aligned_at(newsize, align, offset);
}
let usable = unsafe { usable_size(p) };
if newsize <= usable
&& newsize >= usable / 2
&& crate::bins::is_aligned_at(p.addr(), offset, align)
{
stat_realloc(true);
return p;
}
let np = malloc_aligned_at(newsize, align, offset);
if np.is_null() {
return ptr::null_mut();
}
unsafe {
core::ptr::copy_nonoverlapping(p, np, usable.min(newsize));
free(p);
stat_realloc(false);
}
np
}
pub unsafe fn rezalloc(p: *mut u8, newsize: usize) -> *mut u8 {
unsafe { rezalloc_aligned_at(p, newsize, 1, 0) }
}
pub unsafe fn recalloc(p: *mut u8, newcount: usize, size: usize) -> *mut u8 {
match newcount.checked_mul(size) {
Some(total) => unsafe { rezalloc(p, total) },
None => ptr::null_mut(),
}
}
pub unsafe fn rezalloc_aligned(p: *mut u8, newsize: usize, align: usize) -> *mut u8 {
unsafe { rezalloc_aligned_at(p, newsize, align, 0) }
}
pub unsafe fn rezalloc_aligned_at(
p: *mut u8,
newsize: usize,
align: usize,
offset: usize,
) -> *mut u8 {
if p.is_null() {
return if align <= 1 {
zalloc(newsize)
} else {
zalloc_aligned_at(newsize, align, offset)
};
}
let usable = unsafe { usable_size(p) };
if newsize <= usable
&& newsize >= usable / 2
&& crate::bins::is_aligned_at(p.addr(), offset, align)
{
stat_realloc(true);
return p;
}
let np = if align <= 1 {
malloc(newsize)
} else {
malloc_aligned_at(newsize, align, offset)
};
if np.is_null() {
return ptr::null_mut();
}
unsafe {
let keep = usable.min(newsize);
core::ptr::copy_nonoverlapping(p, np, keep);
let new_usable = usable_size(np);
core::ptr::write_bytes(np.add(keep), 0, new_usable - keep);
free(p);
stat_realloc(false);
}
np
}
pub unsafe fn recalloc_aligned(p: *mut u8, newcount: usize, size: usize, align: usize) -> *mut u8 {
unsafe { recalloc_aligned_at(p, newcount, size, align, 0) }
}
pub unsafe fn recalloc_aligned_at(
p: *mut u8,
newcount: usize,
size: usize,
align: usize,
offset: usize,
) -> *mut u8 {
match newcount.checked_mul(size) {
Some(total) => unsafe { rezalloc_aligned_at(p, total, align, offset) },
None => ptr::null_mut(),
}
}
pub unsafe fn free(p: *mut u8) {
unsafe { free_inline(p) }
}
#[inline(always)]
pub unsafe fn free_inline(p: *mut u8) {
unsafe { free_inline_impl::<true>(p) }
}
#[inline(always)]
pub unsafe fn free_inline_flags_first(p: *mut u8) {
unsafe { free_inline_impl::<false>(p) }
}
#[inline(always)]
unsafe fn free_inline_impl<const OWNER_FIRST: bool>(p: *mut u8) {
if p.is_null() {
return;
}
debug_foreign_pointer_guard(p);
let seg = segment_of(p);
unsafe { free_body::<OWNER_FIRST, false>(p, seg, ptr::null_mut()) }
}
#[inline(always)]
unsafe fn free_body_classic<const OWNER_FIRST: bool, const KNOWN_PAGE: bool>(
p: *mut u8,
seg: *mut Segment,
known_pg: *mut Page,
) {
unsafe {
let owner_tid = if crate::ONE_THREAD {
0
} else {
(*seg).thread_id.load(core::sync::atomic::Ordering::Acquire)
};
#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))]
let local = crate::ONE_THREAD || owner_tid == init::thread_id();
let pg = if KNOWN_PAGE {
known_pg
} else {
page_of(seg, p)
};
let flags = (*pg).flags.load(Ordering::Relaxed);
if OWNER_FIRST {
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
core::arch::asm!(
"cmp {tid}, fs:0",
"jne {remote}",
tid = in(reg) owner_tid,
remote = label {
unsafe {
if flags & pflags::HAS_ALIGNED != 0 {
return remote_free_flagged(pg, p, flags);
}
remote_free(pg, p.cast::<Block>())
};
return;
},
options(nostack, readonly),
);
#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))]
if !local {
if flags & pflags::HAS_ALIGNED != 0 {
return remote_free_flagged(pg, p, flags);
}
remote_free(pg, p.cast::<Block>());
return;
}
}
if flags & pflags::SLOW_FREE == 0 {
if !OWNER_FIRST {
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
core::arch::asm!(
"cmp {tid}, fs:0",
"jne {remote}",
tid = in(reg) owner_tid,
remote = label {
unsafe { remote_free(pg, p.cast::<Block>()) };
return;
},
options(nostack, readonly),
);
#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))]
if !local {
remote_free(pg, p.cast::<Block>());
return;
}
}
free_local_plain(seg, pg, p);
return;
}
if OWNER_FIRST {
return free_flagged_local(p, seg, pg, flags);
}
free_general(p, seg, pg, owner_tid);
}
}
#[cold]
#[inline(never)]
unsafe fn free_flagged_local(p: *mut u8, seg: *mut Segment, pg: *mut Page, flags: u8) {
unsafe {
let block = if flags & pflags::HAS_ALIGNED != 0 && flags & pflags::HUGE_SEGMENT == 0 {
unalign(pg, p)
} else {
p
};
free_local_owned(seg, pg, block);
}
}
#[inline(always)]
unsafe fn free_body<const OWNER_FIRST: bool, const KNOWN_PAGE: bool>(
p: *mut u8,
seg: *mut Segment,
known_pg: *mut Page,
) {
if crate::page::FAST_OWNER {
unsafe {
let pg = if KNOWN_PAGE {
known_pg
} else {
page_of(seg, p)
};
let owner = (*pg).owner_plain.load(Ordering::Relaxed);
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
core::arch::asm!(
"cmp {owner}, fs:0",
"jne {slow}",
owner = in(reg) owner,
slow = label {
unsafe {
if KNOWN_PAGE {
free_slow_known(p, pg)
} else {
free_slow(p, seg, pg)
}
};
return;
},
options(nostack, readonly),
);
#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))]
if owner != init::thread_id() {
return if KNOWN_PAGE {
free_slow_known(p, pg)
} else {
free_slow(p, seg, pg)
};
}
debug_assert_eq!(
(*pg).flags.load(Ordering::Relaxed) & pflags::SLOW_FREE,
0,
"free fast path: the owner word matched a flagged page"
);
debug_assert_eq!(
(*seg).thread_id.load(Ordering::Relaxed),
owner,
"free fast path: the owner word outlived its segment's owner"
);
return free_local_plain(seg, pg, p);
}
}
unsafe { free_body_classic::<OWNER_FIRST, KNOWN_PAGE>(p, seg, known_pg) }
}
#[cold]
#[inline(never)]
unsafe fn free_slow(p: *mut u8, seg: *mut Segment, pg: *mut Page) {
unsafe { free_body_classic::<true, true>(p, seg, pg) }
}
#[cold]
#[inline(never)]
unsafe fn free_slow_known(p: *mut u8, pg: *mut Page) {
unsafe { free_slow(p, segment_of(p), pg) }
}
#[inline(always)]
unsafe fn free_local_plain(seg: *mut Segment, pg: *mut Page, p: *mut u8) {
unsafe {
debug_assert_eq!(
(*seg).kind,
SegmentKind::Normal,
"free fast path: HUGE_SEGMENT clear but segment is Huge"
);
debug_assert_ne!(
(*pg).bin as usize,
BIN_HUGE,
"free fast path: SINGLE_BLOCK clear but bin is BIN_HUGE"
);
crate::page::page_link_local(pg, p.cast::<Block>());
#[cfg(debug_assertions)]
{
(*owner_heap(pg)).stats.frees += 1;
}
#[cfg(all(
target_arch = "x86_64",
not(miri),
not(all(target_env = "msvc", panic = "unwind"))
))]
{
core::arch::asm!(
"sub dword ptr [{pg} + {off}], 1",
"jle {cold}",
pg = in(reg) pg,
off = const crate::page::USED_OFFSET,
cold = label {
unsafe {
if ((*pg).used as usize | (*pg).next.addr() | (*pg).prev.addr())
== 0
{
return;
}
return retire_or_abort(pg);
}
},
options(nostack),
);
}
#[cfg(all(
target_arch = "x86_64",
not(miri),
target_env = "msvc",
panic = "unwind"
))]
{
let le: u8;
core::arch::asm!(
"sub dword ptr [{pg} + {off}], 1",
"setle {le}",
pg = in(reg) pg,
off = const crate::page::USED_OFFSET,
le = out(reg_byte) le,
options(nostack),
);
if le != 0 {
if ((*pg).used as usize | (*pg).next.addr() | (*pg).prev.addr()) == 0 {
return;
}
retire_or_abort(pg);
}
}
#[cfg(not(all(target_arch = "x86_64", not(miri))))]
{
let u = (*pg).used.wrapping_sub(1);
(*pg).used = u;
if (u as i32) <= 0 {
if (u as usize | (*pg).next.addr() | (*pg).prev.addr()) == 0 {
return;
}
retire_or_abort(pg);
}
}
}
}
#[cold]
#[inline(never)]
unsafe fn retire_or_abort(pg: *mut Page) {
if (unsafe { (*pg).used } as i32) < 0 {
crate::page::double_free_abort();
}
if unsafe { (*pg).next.is_null() && (*pg).prev.is_null() } {
return;
}
unsafe {
let seg = segment_of(pg.cast());
(*owner_heap(pg)).retire_emptied(seg, pg);
}
}
#[cold]
#[inline(never)]
unsafe fn remote_free_flagged(pg: *mut Page, p: *mut u8, flags: u8) {
let b = if flags & pflags::HAS_ALIGNED != 0 && flags & pflags::HUGE_SEGMENT == 0 {
unsafe { unalign(pg, p) }
} else {
p
};
unsafe { remote_free(pg, b.cast::<Block>()) }
}
#[cold]
#[inline(never)]
unsafe fn free_general(p: *mut u8, seg: *mut Segment, pg: *mut Page, owner_tid: usize) {
unsafe {
debug_assert_eq!(seg, segment_of(p));
debug_assert_eq!(pg, page_of(seg, p));
let flags = (*pg).flags.load(Ordering::Relaxed);
let block = if flags & pflags::HAS_ALIGNED != 0 && flags & pflags::HUGE_SEGMENT == 0 {
unalign(pg, p)
} else {
p
};
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
{
core::arch::asm!(
"cmp {tid}, fs:0",
"jne {remote}",
tid = in(reg) owner_tid,
remote = label {
unsafe { remote_free(pg, block.cast::<Block>()) };
return;
},
options(nostack, readonly),
);
free_local_owned(seg, pg, block);
}
#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))]
if crate::ONE_THREAD || owner_tid == init::thread_id() {
free_local_owned(seg, pg, block);
} else {
remote_free(pg, block.cast::<Block>());
}
}
}
#[inline]
pub unsafe fn usable_size_plain(p: ptr::NonNull<u8>) -> Option<usize> {
let p = p.as_ptr();
let seg = segment_of(p);
unsafe {
let pg = page_of(seg, p);
let flags = (*pg).flags.load(Ordering::Relaxed);
if flags & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK | pflags::HUGE_SEGMENT) == 0 {
Some((*pg).block_size)
} else {
None
}
}
}
#[inline]
pub unsafe fn usable_size(p: *const u8) -> usize {
if p.is_null() {
return 0;
}
let seg = segment_of(p.cast_mut());
unsafe {
let pg = page_of(seg, p.cast_mut());
let flags = (*pg).flags.load(Ordering::Relaxed);
if flags & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK | pflags::HUGE_SEGMENT) == 0 {
return (*pg).block_size;
}
debug_assert_eq!(
flags & pflags::HUGE_SEGMENT != 0,
(*seg).kind == SegmentKind::Huge,
"usable_size: HUGE_SEGMENT flag disagrees with the segment kind"
);
usable_size_slow(pg, p, flags)
}
}
#[cold]
#[inline(never)]
unsafe fn usable_size_slow(pg: *mut Page, p: *const u8, flags: u8) -> usize {
unsafe {
if flags & pflags::HUGE_SEGMENT != 0 {
return (*pg).block_size;
}
let start = unalign(pg, p.cast_mut());
(*pg).block_size - (p.addr() - start.addr())
}
}
#[inline]
pub unsafe fn realloc(p: *mut u8, newsize: usize) -> *mut u8 {
if p.is_null() {
return malloc_impl::<false, true>(newsize);
}
unsafe { realloc_live(ptr::NonNull::new_unchecked(p), newsize) }
}
#[inline(never)]
unsafe fn realloc_live(p: ptr::NonNull<u8>, newsize: usize) -> *mut u8 {
unsafe { realloc_live_body(p, newsize) }
}
#[inline(never)]
pub unsafe fn realloc_rust(
p: ptr::NonNull<u8>,
_old_size: usize,
_align: usize,
newsize: usize,
) -> *mut u8 {
unsafe { realloc_live_body(p, newsize) }
}
const LEAF_MOVE: bool =
crate::page::FAST_OWNER && cfg!(all(target_arch = "x86_64", target_os = "linux", not(miri)));
#[inline(always)]
unsafe fn realloc_live_body(p: ptr::NonNull<u8>, newsize: usize) -> *mut u8 {
if !LEAF_MOVE {
return unsafe { realloc_live_classic(p, newsize) };
}
let p = p.as_ptr();
let seg = segment_of(p);
unsafe {
let pg = page_of(seg, p);
let flags = (*pg).flags.load(Ordering::Relaxed);
if flags & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK | pflags::HUGE_SEGMENT) != 0 {
return realloc_flagged(p, newsize);
}
let usable = (*pg).block_size;
let copy = if newsize > usable {
usable
} else if newsize >= usable / 2 {
stat_realloc(true);
return p;
} else {
newsize
};
if copy > 64 {
return realloc_move_out::<true>(p, newsize, copy, pg);
}
if newsize <= SMALL_SIZE_MAX {
let hb = init::heap_box_fast();
let h = (*hb).heap.get();
let np = crate::page::page_pop((*h).direct[crate::types::wsize_from_size(newsize)]);
if !np.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
copy_small(p, np, copy);
stat_realloc(false);
return free_moved(p, pg, np);
}
}
realloc_move_out::<false>(p, newsize, copy, pg)
}
}
#[cold]
#[inline(never)]
unsafe fn realloc_flagged(p: *mut u8, newsize: usize) -> *mut u8 {
unsafe { realloc_live_classic(ptr::NonNull::new_unchecked(p), newsize) }
}
#[inline(never)]
unsafe fn realloc_move_out<const BIG: bool>(
p: *mut u8,
newsize: usize,
copy: usize,
pg: *mut Page,
) -> *mut u8 {
let np = malloc_impl::<false, false>(newsize);
if np.is_null() {
return ptr::null_mut();
}
unsafe {
if BIG {
core::ptr::copy_nonoverlapping(p, np, copy);
} else {
copy_block(p, np, copy);
}
free_body::<true, true>(p, segment_of(p), pg);
stat_realloc(false);
}
np
}
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
#[inline(always)]
unsafe fn free_moved(p: *mut u8, pg: *mut Page, np: *mut u8) -> *mut u8 {
unsafe {
let owner = (*pg).owner_plain.load(Ordering::Relaxed);
core::arch::asm!(
"cmp {owner}, fs:0",
"jne {slow}",
owner = in(reg) owner,
slow = label {
return unsafe { free_slow_ret(p, pg, np) };
},
options(nostack, readonly),
);
crate::page::page_link_local(pg, p.cast::<Block>());
#[cfg(debug_assertions)]
{
(*owner_heap(pg)).stats.frees += 1;
}
core::arch::asm!(
"sub dword ptr [{pg} + {off}], 1",
"jle {cold}",
pg = in(reg) pg,
off = const crate::page::USED_OFFSET,
cold = label {
unsafe {
if ((*pg).used as usize | (*pg).next.addr() | (*pg).prev.addr()) == 0 {
return np;
}
return retire_ret(pg, np);
}
},
options(nostack),
);
np
}
}
#[cfg(not(all(target_arch = "x86_64", target_os = "linux", not(miri))))]
#[inline(always)]
unsafe fn free_moved(p: *mut u8, pg: *mut Page, np: *mut u8) -> *mut u8 {
unsafe { free_body::<true, true>(p, segment_of(p), pg) };
np
}
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
#[cold]
#[inline(never)]
unsafe fn free_slow_ret(p: *mut u8, pg: *mut Page, np: *mut u8) -> *mut u8 {
unsafe { free_slow_known(p, pg) };
core::hint::black_box(np)
}
#[cfg(all(target_arch = "x86_64", target_os = "linux", not(miri)))]
#[cold]
#[inline(never)]
unsafe fn retire_ret(pg: *mut Page, np: *mut u8) -> *mut u8 {
unsafe { retire_or_abort(pg) };
core::hint::black_box(np)
}
#[inline(always)]
unsafe fn realloc_live_classic(p: ptr::NonNull<u8>, newsize: usize) -> *mut u8 {
let p = p.as_ptr();
let seg = segment_of(p);
let (pg, usable) = unsafe {
let pg = page_of(seg, p);
let flags = (*pg).flags.load(Ordering::Relaxed);
let usable =
if flags & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK | pflags::HUGE_SEGMENT) == 0 {
(*pg).block_size
} else {
usable_size_slow(pg, p, flags)
};
(pg, usable)
};
let copy = if newsize > usable {
usable
} else if newsize >= usable / 2 {
stat_realloc(true);
return p;
} else {
newsize
};
unsafe { realloc_move_at(p, newsize, copy, seg, pg) }
}
#[inline(always)]
unsafe fn realloc_move_at(
p: *mut u8,
newsize: usize,
copy: usize,
seg: *mut Segment,
pg: *mut Page,
) -> *mut u8 {
let np = malloc_impl::<false, false>(newsize);
if np.is_null() {
return ptr::null_mut();
}
unsafe {
copy_block(p, np, copy);
free_body::<true, true>(p, seg, pg);
stat_realloc(false);
}
np
}
#[inline(always)]
unsafe fn copy_block(src: *const u8, dst: *mut u8, n: usize) {
unsafe {
if n <= 64 {
copy_small(src, dst, n);
} else {
core::ptr::copy_nonoverlapping(src, dst, n);
}
}
}
#[inline(always)]
unsafe fn copy_small(src: *const u8, dst: *mut u8, n: usize) {
debug_assert!(n <= 64, "copy_small: the ladder stops at 64 bytes");
unsafe {
if n > 32 {
core::ptr::copy_nonoverlapping(src, dst, 32);
core::ptr::copy_nonoverlapping(src.add(n - 32), dst.add(n - 32), 32);
} else if n >= 16 {
core::ptr::copy_nonoverlapping(src, dst, 16);
core::ptr::copy_nonoverlapping(src.add(n - 16), dst.add(n - 16), 16);
} else if n >= 8 {
core::ptr::copy_nonoverlapping(src, dst, 8);
core::ptr::copy_nonoverlapping(src.add(n - 8), dst.add(n - 8), 8);
} else if n >= 4 {
core::ptr::copy_nonoverlapping(src, dst, 4);
core::ptr::copy_nonoverlapping(src.add(n - 4), dst.add(n - 4), 4);
} else if n >= 2 {
core::ptr::copy_nonoverlapping(src, dst, 2);
core::ptr::copy_nonoverlapping(src.add(n - 2), dst.add(n - 2), 2);
} else if n == 1 {
*dst = *src;
}
}
}
pub unsafe fn reallocn(p: *mut u8, count: usize, size: usize) -> *mut u8 {
match count.checked_mul(size) {
Some(total) => unsafe { realloc(p, total) },
None => ptr::null_mut(),
}
}
pub unsafe fn reallocf(p: *mut u8, newsize: usize) -> *mut u8 {
let np = unsafe { realloc(p, newsize) };
if np.is_null() && !p.is_null() {
unsafe { free(p) };
}
np
}
pub unsafe fn expand(p: *mut u8, newsize: usize) -> *mut u8 {
if p.is_null() {
return ptr::null_mut();
}
let usable = unsafe { usable_size(p) };
if newsize <= usable {
p
} else {
ptr::null_mut()
}
}
pub fn is_in_heap_region(p: *const u8) -> bool {
crate::segment_map::contains(p)
}
pub fn collect(force: bool) {
#[cfg(ra_shared_light_heap)]
if init::heap_box_fast() == init::empty_heap_box_ptr() && crate::shared_light::collect(force) {
return;
}
let h = my_heap();
if h.is_null() {
return; }
unsafe { (*h).collect(force) };
}
pub fn stats() -> crate::heap::Stats {
let h = my_heap();
if h.is_null() {
return crate::heap::Stats::new(); }
unsafe { (*h).stats }
}
#[inline]
unsafe fn heap_of(hb: *mut init::HeapBox) -> *mut Heap {
unsafe {
debug_assert_eq!(
(*hb).owner_tid,
init::thread_id(),
"heap used off its owning thread"
);
(*hb).heap.get()
}
}
pub unsafe fn heap_malloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe {
let h = heap_of(hb);
fire_deferred_if_due(h);
(*h).malloc(size).0
}
}
pub unsafe fn heap_zalloc(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe {
let h = heap_of(hb);
fire_deferred_if_due(h);
let (p, is_zero) = (*h).malloc(size);
if !p.is_null() {
zero_block(p, is_zero);
}
p
}
}
pub unsafe fn heap_malloc_aligned_at(
hb: *mut init::HeapBox,
size: usize,
align: usize,
offset: usize,
) -> *mut u8 {
unsafe {
let h = heap_of(hb);
fire_deferred_if_due(h);
(*h).malloc_aligned_at(size, align, offset).0
}
}
pub unsafe fn heap_zalloc_aligned_at(
hb: *mut init::HeapBox,
size: usize,
align: usize,
offset: usize,
) -> *mut u8 {
unsafe {
let h = heap_of(hb);
fire_deferred_if_due(h);
let (p, is_zero) = (*h).malloc_aligned_at(size, align, offset);
if !p.is_null() {
zero_block(p, is_zero);
}
p
}
}
pub unsafe fn heap_realloc(hb: *mut init::HeapBox, p: *mut u8, newsize: usize) -> *mut u8 {
if p.is_null() {
return unsafe { heap_malloc(hb, newsize) };
}
let usable = unsafe { usable_size(p) };
if newsize <= usable && newsize >= usable / 2 {
#[cfg(debug_assertions)]
unsafe {
(*heap_of(hb)).stats.realloc_in_place += 1;
}
return p;
}
unsafe {
let np = heap_malloc(hb, newsize);
if np.is_null() {
return ptr::null_mut();
}
core::ptr::copy_nonoverlapping(p, np, usable.min(newsize));
free(p);
#[cfg(debug_assertions)]
{
(*heap_of(hb)).stats.realloc_moved += 1;
}
np
}
}
#[inline]
pub unsafe fn heap_malloc_fast(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe { heap_malloc_impl::<true>(hb, size) }
}
#[inline(always)]
unsafe fn heap_malloc_impl<const DRY: bool>(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe {
#[cfg(debug_assertions)]
let _ = heap_of(hb);
if size <= SMALL_SIZE_MAX {
let w = crate::types::wsize_from_size(size);
let h = (*hb).heap.get();
let p = (*h).direct[w];
let b = crate::page::page_pop(p);
if !b.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
return b;
}
if DRY {
return malloc_dry_hb_first(hb, size, p);
}
}
malloc_slow_hb_first(hb, size)
}
}
#[inline]
pub unsafe fn heap_zalloc_fast(hb: *mut init::HeapBox, size: usize) -> *mut u8 {
unsafe {
#[cfg(debug_assertions)]
let _ = heap_of(hb);
if size <= SMALL_SIZE_MAX {
let w = crate::types::wsize_from_size(size);
let h = (*hb).heap.get();
let p = (*h).direct[w];
let b = crate::page::page_pop(p);
if !b.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
if (*p).free_is_zero {
b.cast::<usize>().write(0);
} else {
ptr::write_bytes(b, 0, (*p).block_size);
}
return b;
}
return zalloc_dry(hb, size, p);
}
zalloc_slow(hb, size)
}
}
#[inline(never)]
pub unsafe fn heap_realloc_fast(
hb: *mut init::HeapBox,
p: ptr::NonNull<u8>,
newsize: usize,
) -> *mut u8 {
let p = p.as_ptr();
if !LEAF_MOVE {
return unsafe { heap_realloc(hb, p, newsize) };
}
let seg = segment_of(p);
unsafe {
#[cfg(debug_assertions)]
let _ = heap_of(hb);
let pg = page_of(seg, p);
let flags = (*pg).flags.load(Ordering::Relaxed);
if flags & (pflags::HAS_ALIGNED | pflags::SINGLE_BLOCK | pflags::HUGE_SEGMENT) != 0 {
return heap_realloc_flagged(hb, p, newsize);
}
let usable = (*pg).block_size;
let copy = if newsize > usable {
usable
} else if newsize >= usable / 2 {
stat_realloc(true);
return p;
} else {
newsize
};
if copy > 64 {
return heap_realloc_move_out::<true>(hb, p, newsize, copy, pg);
}
if newsize <= SMALL_SIZE_MAX {
let h = (*hb).heap.get();
let np = crate::page::page_pop((*h).direct[crate::types::wsize_from_size(newsize)]);
if !np.is_null() {
#[cfg(debug_assertions)]
{
(*h).stats.allocs += 1;
}
copy_small(p, np, copy);
stat_realloc(false);
return free_moved(p, pg, np);
}
}
heap_realloc_move_out::<false>(hb, p, newsize, copy, pg)
}
}
#[cold]
#[inline(never)]
unsafe fn heap_realloc_flagged(hb: *mut init::HeapBox, p: *mut u8, newsize: usize) -> *mut u8 {
unsafe { heap_realloc(hb, p, newsize) }
}
#[inline(never)]
unsafe fn heap_realloc_move_out<const BIG: bool>(
hb: *mut init::HeapBox,
p: *mut u8,
newsize: usize,
copy: usize,
pg: *mut Page,
) -> *mut u8 {
let np = unsafe { heap_malloc_impl::<false>(hb, newsize) };
if np.is_null() {
return ptr::null_mut();
}
unsafe {
if BIG {
core::ptr::copy_nonoverlapping(p, np, copy);
} else {
copy_block(p, np, copy);
}
free_body::<true, true>(p, segment_of(p), pg);
stat_realloc(false);
}
np
}
pub unsafe fn heap_collect(hb: *mut init::HeapBox, force: bool) {
unsafe { (*heap_of(hb)).collect(force) };
}
pub unsafe fn heap_contains_block(hb: *mut init::HeapBox, p: *const u8) -> bool {
if p.is_null() {
return false;
}
let target = segment_of(p.cast_mut());
unsafe {
let h = heap_of(hb);
let mut seg = (*h).segments;
while !seg.is_null() {
if seg == target {
return true;
}
seg = (*seg).next;
}
let mut seg = (*h).huge_segments;
while !seg.is_null() {
if seg == target {
return true;
}
seg = (*seg).next;
}
}
false
}
pub unsafe fn heap_check_owned(hb: *mut init::HeapBox, p: *const u8) -> bool {
if !unsafe { heap_contains_block(hb, p) } {
return false;
}
let seg = segment_of(p.cast_mut());
unsafe {
if (*seg).kind == SegmentKind::Huge {
return true;
}
let pg = page_of(seg, p.cast_mut());
(*pg).block_size > 0
}
}
pub fn check_owned(p: *const u8) -> bool {
is_in_heap_region(p)
}