use core::ptr::NonNull;
use crate::class::{self, NCLASSES};
use crate::os;
use crate::outbound::Outbound;
use crate::segment::{self, SEGMENT_BYTES, Segment};
use crate::spanlist::BINS;
pub const PER_CLASS_CAP: u32 = 16_777_216;
#[derive(Debug)]
pub struct Heap {
id: usize,
pub(crate) segments: *mut Segment,
pub(crate) partial: [Option<(NonNull<Segment>, u8)>; NCLASSES],
pub(crate) spans_in_class: [u32; NCLASSES],
pub(crate) class_live: [u32; NCLASSES],
pub(crate) live_bytes: u64,
pub(crate) outbound: Outbound,
pub(crate) bins: [[usize; BINS]; NCLASSES],
pub(crate) free_spans: usize,
claims: [Option<Claim>; NCLASSES],
class_cap: u32,
pub(crate) token: usize,
pub(crate) tally: crate::tally::Tally,
pub(crate) parked: *const segment::ForeignTally,
pub(crate) empty_in_class: [u32; NCLASSES],
pub(crate) epoch: u32,
pub(crate) wheel: [usize; crate::purge::WHEEL],
pub(crate) purge_delay: u32,
#[cfg(test)]
pub(crate) discards: u64,
}
impl Heap {
#[must_use]
pub const fn new(id: usize) -> Self {
Self::with_class_cap(id, PER_CLASS_CAP)
}
#[must_use]
pub const fn with_class_cap(id: usize, class_cap: u32) -> Self {
Self {
id,
segments: core::ptr::null_mut(),
partial: [None; NCLASSES],
spans_in_class: [0; NCLASSES],
class_live: [0; NCLASSES],
live_bytes: 0,
outbound: Outbound::new(),
bins: [[0; BINS]; NCLASSES],
free_spans: 0,
claims: [None; NCLASSES],
class_cap,
token: 0,
tally: crate::tally::Tally::NEW,
parked: core::ptr::null(),
empty_in_class: [0; NCLASSES],
epoch: 0,
wheel: [0; crate::purge::WHEEL],
purge_delay: crate::purge::PURGE_DELAY,
#[cfg(test)]
discards: 0,
}
}
pub fn ensure_identity(&mut self) {
if self.id == 0 {
self.id = core::ptr::from_mut(self) as usize;
}
}
#[inline]
pub fn alloc(&mut self, size: usize, align: usize) -> Option<NonNull<u8>> {
match class::index_of(size, align) {
Some(c) => self.alloc_small(c, size),
None => self.alloc_large(size, align),
}
}
pub unsafe fn try_resize_in_place(
&mut self,
ptr: NonNull<u8>,
old_size: usize,
new_size: usize,
align: usize,
) -> bool {
let (Some(a), Some(b)) =
(class::index_of(old_size, align), class::index_of(new_size, align))
else {
return false;
};
if a != b {
return false;
}
let seg = unsafe { segment::segment_of(ptr) };
if unsafe { seg.as_ref() }.owner != self.id {
return false;
}
self.live_bytes = self.live_bytes - old_size as u64 + new_size as u64;
true
}
#[inline]
pub unsafe fn dealloc(&mut self, ptr: NonNull<u8>, size: usize, align: usize) {
match class::index_of(size, align) {
Some(c) => unsafe { self.dealloc_small(ptr, c, size) },
None => unsafe { self.dealloc_large(ptr, size) },
}
}
#[inline]
pub(crate) fn alloc_small(&mut self, c: usize, size: usize) -> Option<NonNull<u8>> {
let Some(slot) = self.pop_claimed(c) else {
return self.alloc_refill(c, size);
};
self.live_bytes += size as u64;
Some(slot)
}
#[cold]
#[inline(never)]
fn alloc_refill(&mut self, c: usize, size: usize) -> Option<NonNull<u8>> {
let slot = self.pop_slot(c).or_else(|| self.slow_path(c))?;
self.live_bytes += size as u64;
Some(slot)
}
fn slow_path(&mut self, c: usize) -> Option<NonNull<u8>> {
for drained in [false, true] {
if drained {
self.drain_foreign();
}
if let Some((seg, ix)) = self.take_densest(c) {
self.partial[c] = Some((seg, ix as u8));
if let Some(p) = self.pop_slot(c) {
return Some(p);
}
}
}
self.claim_span(c)?;
self.pop_slot(c)
}
fn pop_slot(&mut self, c: usize) -> Option<NonNull<u8>> {
self.refill_claim(c)?;
self.pop_claimed(c)
}
fn claim_span(&mut self, c: usize) -> Option<()> {
if self.spans_in_class[c] >= self.class_cap {
return None;
}
let (seg, ix) = match self.pop_free_span() {
Some(s) => s,
None => {
self.map_segment()?;
self.pop_free_span()?
}
};
let meta = unsafe { &mut (*seg.as_ptr()).spans[ix] };
self.tally.span_claimed(meta);
meta.reset(c as u8);
self.spans_in_class[c] += 1;
self.empty_in_class[c] += 1;
self.partial[c] = Some((seg, ix as u8));
Some(())
}
fn map_segment(&mut self) -> Option<()> {
let base = os::map_aligned(SEGMENT_BYTES, SEGMENT_BYTES)?;
if self.token == 0 {
self.token = crate::rtree::new_token();
}
if !crate::rtree::set(base.as_ptr() as usize, self.token) {
unsafe { os::unmap(base, SEGMENT_BYTES) };
return None;
}
let seg = unsafe { Segment::init(base, self.id) };
unsafe {
if self.parked.is_null() {
self.parked = (*seg.as_ptr()).home;
}
(*seg.as_ptr()).home = self.parked;
(*seg.as_ptr()).next = self.segments;
}
self.segments = seg.as_ptr();
self.tally.segment_mapped();
self.file_new_segment(seg);
Some(())
}
#[inline(never)]
fn alloc_large(&mut self, size: usize, align: usize) -> Option<NonNull<u8>> {
crate::large::alloc(size, align)
}
#[inline(never)]
unsafe fn dealloc_large(&mut self, ptr: NonNull<u8>, size: usize) {
unsafe { crate::large::dealloc(ptr, size) };
}
}
#[path = "heap_claims.rs"]
mod heap_claims;
#[path = "heap_defrag.rs"]
mod heap_defrag;
#[path = "heap_free.rs"]
mod heap_free;
pub(crate) use heap_claims::Claim;