use std::alloc::{Layout, alloc, dealloc};
use std::mem::align_of;
use std::ptr::NonNull;
use crate::gc::gc_ptr::{GcObject, GcPtr};
pub use crate::objects::heap_object::HeapObject;
pub struct MemoryRegion {
base: *mut u8,
capacity: usize,
used: usize,
}
unsafe impl Send for MemoryRegion {}
impl MemoryRegion {
pub fn new(capacity: usize) -> Self {
assert!(capacity > 0, "region capacity must be non-zero");
let layout = Layout::from_size_align(capacity, 8).expect("valid layout");
let base = unsafe { alloc(layout) };
assert!(!base.is_null(), "heap region allocation failed");
Self {
base,
capacity,
used: 0,
}
}
pub fn bump_alloc(&mut self, layout: Layout) -> Option<*mut u8> {
let align = layout.align();
let size = layout.size();
let current = self.base as usize + self.used;
let aligned = current.checked_add(align - 1)? & !(align - 1);
let end = aligned.checked_add(size)?;
let new_used = end - self.base as usize;
if new_used > self.capacity {
return None;
}
self.used = new_used;
Some(aligned as *mut u8)
}
pub fn reset(&mut self) {
self.used = 0;
}
pub fn used(&self) -> usize {
self.used
}
pub fn capacity(&self) -> usize {
self.capacity
}
pub(crate) fn base_ptr(&self) -> *mut u8 {
self.base
}
pub(crate) fn force_used(&mut self, used: usize) {
assert!(
used <= self.capacity,
"force_used: used ({used}) must not exceed capacity ({})",
self.capacity
);
self.used = used;
}
}
impl Drop for MemoryRegion {
fn drop(&mut self) {
if !self.base.is_null() {
let layout = Layout::from_size_align(self.capacity, 8).expect("valid layout");
unsafe { dealloc(self.base, layout) };
}
}
}
const YOUNG_SEMI_SPACE_SIZE: usize = 4 * 1024 * 1024;
const OLD_SPACE_SIZE: usize = 64 * 1024 * 1024;
const LARGE_OBJECT_THRESHOLD: usize = YOUNG_SEMI_SPACE_SIZE;
pub struct SemiSpace {
from_space: MemoryRegion,
to_space: MemoryRegion,
}
impl SemiSpace {
pub fn new(semi_size: usize) -> Self {
Self {
from_space: MemoryRegion::new(semi_size),
to_space: MemoryRegion::new(semi_size),
}
}
pub fn bump_alloc(&mut self, layout: Layout) -> Option<*mut u8> {
self.from_space.bump_alloc(layout)
}
pub(crate) fn bump_alloc_to_space(&mut self, layout: Layout) -> Option<*mut u8> {
self.to_space.bump_alloc(layout)
}
pub fn is_in_from_space(&self, ptr: *mut HeapObject) -> bool {
let base = self.from_space.base_ptr() as usize;
let end = base + self.from_space.capacity();
let addr = ptr as usize;
addr >= base && addr < end
}
pub(crate) fn to_space_used(&self) -> usize {
self.to_space.used()
}
pub(crate) fn to_space_base(&self) -> *mut u8 {
self.to_space.base_ptr()
}
pub fn collect(&mut self) {
std::mem::swap(&mut self.from_space, &mut self.to_space);
self.to_space.reset();
}
pub fn used(&self) -> usize {
self.from_space.used()
}
pub fn capacity(&self) -> usize {
self.from_space.capacity()
}
}
pub struct OldSpace {
region: MemoryRegion,
}
impl OldSpace {
pub fn new(capacity: usize) -> Self {
Self {
region: MemoryRegion::new(capacity),
}
}
pub fn bump_alloc(&mut self, layout: Layout) -> Option<*mut u8> {
self.region.bump_alloc(layout)
}
pub fn used(&self) -> usize {
self.region.used()
}
pub fn capacity(&self) -> usize {
self.region.capacity()
}
pub(crate) fn base_ptr(&self) -> *mut u8 {
self.region.base_ptr()
}
pub(crate) fn contains(&self, ptr: *mut u8) -> bool {
let base = self.region.base_ptr() as usize;
let end = base + self.region.used();
let addr = ptr as usize;
addr >= base && addr < end
}
pub(crate) fn force_used(&mut self, used: usize) {
self.region.force_used(used);
}
}
pub struct LargeObjectSpace {
objects: Vec<(*mut u8, Layout)>,
}
unsafe impl Send for LargeObjectSpace {}
impl LargeObjectSpace {
pub fn new() -> Self {
Self {
objects: Vec::new(),
}
}
pub fn allocate(&mut self, layout: Layout) -> *mut HeapObject {
let ptr = unsafe { alloc(layout) };
if ptr.is_null() {
return std::ptr::null_mut();
}
unsafe { std::ptr::write_bytes(ptr, 0, layout.size()) };
unsafe { (*(ptr as *mut HeapObject)).init_alloc_size(layout.size() as u32) };
self.objects.push((ptr, layout));
ptr as *mut HeapObject
}
pub fn object_count(&self) -> usize {
self.objects.len()
}
}
impl Default for LargeObjectSpace {
fn default() -> Self {
Self::new()
}
}
impl Drop for LargeObjectSpace {
fn drop(&mut self) {
for (ptr, layout) in self.objects.drain(..) {
unsafe { dealloc(ptr, layout) };
}
}
}
pub struct Heap {
pub young_space: SemiSpace,
pub old_space: OldSpace,
pub large_object_space: LargeObjectSpace,
}
impl Heap {
pub fn new() -> Self {
Self {
young_space: SemiSpace::new(YOUNG_SEMI_SPACE_SIZE),
old_space: OldSpace::new(OLD_SPACE_SIZE),
large_object_space: LargeObjectSpace::new(),
}
}
pub fn allocate(&mut self, layout: Layout) -> *mut HeapObject {
let layout = layout
.align_to(align_of::<HeapObject>())
.expect("alignment adjustment failed")
.pad_to_align();
if layout.size() >= LARGE_OBJECT_THRESHOLD {
return self.large_object_space.allocate(layout);
}
if let Some(ptr) = self.young_space.bump_alloc(layout) {
unsafe { std::ptr::write_bytes(ptr, 0, layout.size()) };
unsafe { (*(ptr as *mut HeapObject)).init_alloc_size(layout.size() as u32) };
return ptr as *mut HeapObject;
}
self.collect();
match self.young_space.bump_alloc(layout) {
Some(ptr) => {
unsafe { std::ptr::write_bytes(ptr, 0, layout.size()) };
unsafe { (*(ptr as *mut HeapObject)).init_alloc_size(layout.size() as u32) };
ptr as *mut HeapObject
}
None => std::ptr::null_mut(),
}
}
pub fn collect(&mut self) {
self.young_space.collect();
}
pub unsafe fn scavenge_with_roots(
&mut self,
roots: &mut [*mut *mut HeapObject],
remembered_set: &mut crate::gc::scavenger::RememberedSet,
) {
unsafe {
crate::gc::scavenger::Scavenger::new(&mut self.young_space, &mut self.old_space)
.scavenge(roots, remembered_set);
}
}
pub unsafe fn full_gc_with_roots(
&mut self,
roots: &mut [*mut *mut HeapObject],
remembered_set: &mut crate::gc::scavenger::RememberedSet,
) {
unsafe { self.scavenge_with_roots(roots, remembered_set) };
unsafe {
crate::gc::mark_sweep_compact::MarkSweepCompactor::new(&mut self.old_space)
.collect(roots)
};
}
pub fn alloc<T: GcObject>(&mut self, value: T) -> Option<GcPtr<T>> {
let layout = Layout::new::<T>();
let raw = self.allocate(layout);
if raw.is_null() {
return None;
}
let typed = raw as *mut T;
unsafe { typed.write(value) };
let padded = layout
.align_to(align_of::<HeapObject>())
.expect("HeapObject alignment is valid")
.pad_to_align();
unsafe { (*raw).init_alloc_size(padded.size() as u32) };
Some(unsafe { GcPtr::from_raw(NonNull::new_unchecked(typed)) })
}
}
impl Default for Heap {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn allocate_returns_non_null_for_small_object() {
let mut heap = Heap::new();
let layout = Layout::new::<HeapObject>();
let ptr = heap.allocate(layout);
assert!(!ptr.is_null());
}
#[test]
fn collect_resets_young_space() {
let mut heap = Heap::new();
let layout = Layout::new::<HeapObject>();
heap.allocate(layout);
let used_before = heap.young_space.used();
assert!(used_before > 0);
heap.collect();
assert_eq!(heap.young_space.used(), 0);
}
#[test]
fn allocate_triggers_minor_gc_and_retries_when_young_space_full() {
let mut heap = Heap::new();
let layout = Layout::new::<HeapObject>();
while heap.young_space.bump_alloc(layout).is_some() {}
let ptr = heap.allocate(layout);
assert!(
!ptr.is_null(),
"allocate must succeed after implicit minor GC"
);
}
#[test]
fn semi_space_collect_resets_from_space() {
let mut ss = SemiSpace::new(1024);
let layout = Layout::from_size_align(64, 8).unwrap();
assert!(ss.bump_alloc(layout).is_some());
assert!(ss.used() > 0);
ss.collect();
assert_eq!(ss.used(), 0);
assert!(ss.bump_alloc(layout).is_some());
}
#[test]
fn old_space_bump_alloc_and_capacity() {
let mut os = OldSpace::new(4096);
let layout = Layout::from_size_align(64, 8).unwrap();
assert!(os.bump_alloc(layout).is_some());
assert_eq!(os.used(), 64);
assert_eq!(os.capacity(), 4096);
}
#[test]
fn large_object_space_allocates_and_tracks_objects() {
let mut los = LargeObjectSpace::new();
let layout = Layout::from_size_align(1024, 8).unwrap();
let ptr = los.allocate(layout);
assert!(!ptr.is_null());
assert_eq!(los.object_count(), 1);
}
#[test]
fn large_object_bypasses_young_space() {
let mut heap = Heap::new();
let large_layout = Layout::from_size_align(LARGE_OBJECT_THRESHOLD, 8).unwrap();
let young_used_before = heap.young_space.used();
let ptr = heap.allocate(large_layout);
assert!(!ptr.is_null());
assert_eq!(
heap.young_space.used(),
young_used_before,
"young space must be untouched for large objects"
);
assert_eq!(heap.large_object_space.object_count(), 1);
}
#[test]
fn allocate_until_young_full_then_verify_after_collection() {
let mut heap = Heap::new();
let layout = Layout::new::<HeapObject>();
while heap.young_space.bump_alloc(layout).is_some() {}
assert!(
heap.young_space.used() > 0,
"from-space must be non-empty after fill"
);
heap.collect();
assert_eq!(
heap.young_space.used(),
0,
"young space must be empty after collection"
);
let ptr = heap.allocate(layout);
assert!(!ptr.is_null(), "allocation after collection must succeed");
}
}