use crate::prelude::*;
#[derive(Clone, Copy, Default)]
struct ChunkBitmask {
bits: [u64; 4],
}
impl ChunkBitmask {
#[inline]
fn set(&mut self, index: usize) {
debug_assert!(index < 256);
let word = index >> 6; let bit = index & 63; unsafe {
*self.bits.get_unchecked_mut(word) |= 1 << bit;
}
}
#[inline]
fn get(&self, index: usize) -> bool {
debug_assert!(index < 256);
let word = index >> 6; let bit = index & 63; unsafe { (*self.bits.get_unchecked(word) & (1 << bit)) != 0 }
}
#[inline]
fn clear(&mut self) {
self.bits = [0; 4];
}
#[inline]
fn iter_unmarked(&self, len: usize) -> impl Iterator<Item = usize> + '_ {
UnmarkedIter {
bitmask: self,
len,
current_word: 0,
current_bits: unsafe { !*self.bits.get_unchecked(0) },
base_index: 0,
}
}
}
struct UnmarkedIter<'a> {
bitmask: &'a ChunkBitmask,
len: usize,
current_word: usize,
current_bits: u64, base_index: usize,
}
impl Iterator for UnmarkedIter<'_> {
type Item = usize;
#[inline]
fn next(&mut self) -> Option<usize> {
loop {
if self.current_bits != 0 {
let bit_pos = self.current_bits.trailing_zeros() as usize;
let index = self.base_index + bit_pos;
self.current_bits &= self.current_bits - 1;
if index < self.len {
return Some(index);
}
}
self.current_word += 1;
if self.current_word >= 4 {
return None;
}
self.base_index = self.current_word << 6; if self.base_index >= self.len {
return None;
}
self.current_bits = unsafe { !*self.bitmask.bits.get_unchecked(self.current_word) };
}
}
}
pub struct Gc<T: Default + Reset + Traceable> {
ptr: NonNull<GcBox<T>>,
space: Weak<RefCell<Space<T>>>,
}
impl<T: Default + Reset + Traceable> PartialEq for Gc<T> {
fn eq(&self, other: &Self) -> bool {
self.ptr == other.ptr
}
}
impl<T: Default + Reset + Traceable> Hash for Gc<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.ptr.hash(state);
}
}
impl<T: Default + Reset + Traceable> Eq for Gc<T> {}
impl<T: Default + Reset + Traceable> Gc<T> {
pub fn borrow(&self) -> Ref<'_, T> {
unsafe { self.ptr.as_ref().data.borrow() }
}
pub fn borrow_mut(&self) -> RefMut<'_, T> {
unsafe { self.ptr.as_ref().data.borrow_mut() }
}
pub fn id(&self) -> usize {
self.ptr.as_ptr() as usize
}
pub fn ptr_eq(a: &Gc<T>, b: &Gc<T>) -> bool {
a.ptr == b.ptr
}
pub fn copy_ref(&self) -> GcPtr<T> {
GcPtr { ptr: self.ptr }
}
}
pub struct GcPtr<T: Default + Reset + Traceable> {
pub(crate) ptr: NonNull<GcBox<T>>,
}
impl<T: Default + Reset + Traceable> Copy for GcPtr<T> {}
impl<T: Default + Reset + Traceable> Clone for GcPtr<T> {
fn clone(&self) -> Self {
*self
}
}
impl<T: Default + Reset + Traceable> Clone for Gc<T> {
fn clone(&self) -> Self {
if let Some(_space) = self.space.upgrade() {
let gc_box = unsafe { self.ptr.as_ref() };
if !gc_box.pooled.get() {
gc_box.ref_count.set(gc_box.ref_count.get() + 1);
}
}
Self {
ptr: self.ptr,
space: self.space.clone(),
}
}
}
impl<T: Default + Reset + Traceable> Drop for Gc<T> {
fn drop(&mut self) {
let Some(space_rc) = self.space.upgrade() else {
return; };
let gc_box = unsafe { self.ptr.as_ref() };
if gc_box.pooled.get() {
return;
}
let count = gc_box.ref_count.get();
if count > 0 {
gc_box.ref_count.set(count - 1);
}
if gc_box.ref_count.get() == 0 {
if let Ok(mut space) = space_rc.try_borrow_mut() {
gc_box.data.borrow_mut().reset();
space.pool_object(gc_box.index, self.ptr);
}
}
}
}
impl<T: Default + Reset + Traceable> fmt::Debug for Gc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Gc").field("ptr", &self.ptr).finish()
}
}
pub trait Traceable: Sized + Default + Reset {
fn trace<F: FnMut(GcPtr<Self>)>(&self, visitor: F);
}
pub trait Reset: Default {
fn reset(&mut self);
}
pub struct GcBox<T: Default + Reset + Traceable> {
index: usize,
data: RefCell<T>,
ref_count: Cell<usize>,
pooled: Cell<bool>,
}
impl<T: Default + Reset + Traceable> GcBox<T> {
fn new(index: usize, data: T) -> Self {
Self {
index,
data: RefCell::new(data),
ref_count: Cell::new(0),
pooled: Cell::new(false),
}
}
}
struct Space<T: Default + Reset + Traceable> {
chunks: Vec<Vec<GcBox<T>>>,
free_list: Vec<NonNull<GcBox<T>>>,
marked_chunks: Vec<ChunkBitmask>,
mark_stack: Vec<NonNull<GcBox<T>>>,
sweep_buffer: Vec<NonNull<GcBox<T>>>,
guard_pool: Vec<Vec<NonNull<GcBox<T>>>>,
active_guards: Vec<Weak<GuardInner<T>>>,
net_allocs: isize,
gc_threshold: isize,
self_weak: Weak<RefCell<Space<T>>>,
}
const DEFAULT_GC_THRESHOLD: usize = 100;
const CHUNK_CAPACITY: usize = 256;
impl<T: Default + Reset + Traceable> Space<T> {
fn new() -> Self {
Self {
chunks: Vec::new(),
free_list: Vec::new(),
marked_chunks: Vec::new(),
mark_stack: Vec::new(),
sweep_buffer: Vec::new(),
guard_pool: Vec::new(),
active_guards: Vec::new(),
net_allocs: 0,
gc_threshold: DEFAULT_GC_THRESHOLD as isize,
self_weak: Weak::new(),
}
}
fn set_self_weak(&mut self, weak: Weak<RefCell<Space<T>>>) {
self.self_weak = weak;
}
fn create_guard(&mut self) -> Guard<T> {
let inner = if let Some(storage) = self.guard_pool.pop() {
Rc::new(GuardInner::with_storage(storage))
} else {
Rc::new(GuardInner::new())
};
self.active_guards.push(Rc::downgrade(&inner));
Guard::new(self.self_weak.clone(), inner)
}
fn return_guard_to_pool(&mut self, mut guarded: Vec<NonNull<GcBox<T>>>) {
if self.guard_pool.len() < 16 {
guarded.clear();
self.guard_pool.push(guarded);
}
}
fn alloc_internal(&mut self) -> Gc<T> {
self.net_allocs += 1;
if self.gc_threshold > 0 && self.net_allocs >= self.gc_threshold {
self.collect();
}
let ptr = if let Some(ptr) = self.free_list.pop() {
let gc_box = unsafe { ptr.as_ptr().as_mut() };
let gc_box = match gc_box {
Some(b) => b,
None => {
#[allow(clippy::panic)]
{
panic!("GC internal error: invalid pool pointer")
}
}
};
gc_box.data.borrow_mut().reset();
gc_box.ref_count.set(1); gc_box.pooled.set(false);
ptr
} else {
let need_new_chunk = self
.chunks
.last()
.is_none_or(|chunk| chunk.len() >= CHUNK_CAPACITY);
if need_new_chunk {
self.chunks.push(Vec::with_capacity(CHUNK_CAPACITY));
self.marked_chunks.push(ChunkBitmask::default());
}
let chunk_idx = self.chunks.len().saturating_sub(1);
let chunk = match self.chunks.last_mut() {
Some(c) => c,
None => {
#[allow(clippy::panic)]
{
panic!("GC internal error: no chunk after creation")
}
}
};
let index_in_chunk = chunk.len();
let index = chunk_idx * CHUNK_CAPACITY + index_in_chunk;
chunk.push(GcBox::new(index, T::default()));
let gc_box = match chunk.last() {
Some(b) => b,
None => {
#[allow(clippy::panic)]
{
panic!("GC internal error: chunk empty after push")
}
}
};
gc_box.ref_count.set(1); NonNull::from(gc_box)
};
Gc {
ptr,
space: self.self_weak.clone(),
}
}
fn pool_object(&mut self, _object_id: usize, ptr: NonNull<GcBox<T>>) {
let gc_box = unsafe { ptr.as_ref() };
if gc_box.pooled.get() {
return;
}
self.net_allocs -= 1;
gc_box.pooled.set(true);
self.free_list.push(ptr);
}
fn mark(&mut self) {
for bitmask in &mut self.marked_chunks {
bitmask.clear();
}
let mut stack = mem::take(&mut self.mark_stack);
stack.clear();
self.active_guards.retain(|weak| {
if let Some(inner) = weak.upgrade() {
let roots = inner.roots.borrow();
for &ptr in roots.iter() {
let gc_box = unsafe { ptr.as_ref() };
if !gc_box.pooled.get() {
stack.push(ptr);
}
}
true } else {
false }
});
let marked_chunks_ptr = self.marked_chunks.as_mut_ptr();
let marked_chunks_len = self.marked_chunks.len();
while let Some(ptr) = stack.pop() {
let gc_box = unsafe { ptr.as_ref() };
let chunk_idx = gc_box.index / CHUNK_CAPACITY;
let index_in_chunk = gc_box.index % CHUNK_CAPACITY;
if chunk_idx >= marked_chunks_len {
continue;
}
let bitmask = unsafe { &mut *marked_chunks_ptr.add(chunk_idx) };
if bitmask.get(index_in_chunk) {
continue;
}
bitmask.set(index_in_chunk);
let data = gc_box.data.borrow();
data.trace(|child: GcPtr<T>| {
let child_box = unsafe { child.ptr.as_ref() };
let child_chunk_idx = child_box.index / CHUNK_CAPACITY;
let child_index_in_chunk = child_box.index % CHUNK_CAPACITY;
if child_chunk_idx < marked_chunks_len {
let child_bitmask = unsafe { &*marked_chunks_ptr.add(child_chunk_idx) };
if !child_bitmask.get(child_index_in_chunk) && !child_box.pooled.get() {
stack.push(child.ptr);
}
}
});
}
self.mark_stack = stack;
}
fn sweep(&mut self) -> usize {
let mut collected = 0;
let mut to_pool = mem::take(&mut self.sweep_buffer);
to_pool.clear();
for (chunk, bitmask) in self.chunks.iter().zip(self.marked_chunks.iter()) {
for index_in_chunk in bitmask.iter_unmarked(chunk.len()) {
if let Some(gc_box) = chunk.get(index_in_chunk)
&& !gc_box.pooled.get()
{
gc_box.data.borrow_mut().reset();
gc_box.ref_count.set(0);
to_pool.push(NonNull::from(gc_box));
collected += 1;
}
}
}
for ptr in &to_pool {
let gc_box = unsafe { ptr.as_ref() };
self.pool_object(gc_box.index, *ptr);
}
to_pool.clear();
self.sweep_buffer = to_pool;
collected
}
fn collect(&mut self) {
self.mark();
self.sweep();
self.net_allocs = 0;
}
fn force_collect(&mut self) {
self.collect();
}
fn stats(&self) -> GcStats {
let total_objects: usize = self.chunks.iter().map(|c| c.len()).sum();
GcStats {
total_objects,
pooled_objects: self.free_list.len(),
live_objects: total_objects - self.free_list.len(),
}
}
fn set_gc_threshold(&mut self, threshold: usize) {
self.gc_threshold = threshold as isize;
}
}
impl<T: Default + Reset + Traceable> Drop for Space<T> {
fn drop(&mut self) {
for chunk in &self.chunks {
for gc_box in chunk {
gc_box.pooled.set(true);
}
}
}
}
pub struct Heap<T: Default + Reset + Traceable> {
inner: Rc<RefCell<Space<T>>>,
}
impl<T: Default + Reset + Traceable> Heap<T> {
pub fn new() -> Self {
let inner = Rc::new(RefCell::new(Space::new()));
inner.borrow_mut().set_self_weak(Rc::downgrade(&inner));
Self { inner }
}
pub fn create_guard(&self) -> Guard<T> {
self.inner.borrow_mut().create_guard()
}
pub fn stats(&self) -> GcStats {
self.inner.borrow().stats()
}
pub fn collect(&self) {
self.inner.borrow_mut().force_collect();
}
pub fn set_gc_threshold(&self, threshold: usize) {
self.inner.borrow_mut().set_gc_threshold(threshold);
}
}
impl<T: Default + Reset + Traceable> Default for Heap<T> {
fn default() -> Self {
Self::new()
}
}
impl<T: Default + Reset + Traceable> Clone for Heap<T> {
fn clone(&self) -> Self {
Heap {
inner: self.inner.clone(),
}
}
}
struct GuardInner<T: Default + Reset + Traceable> {
roots: RefCell<Vec<NonNull<GcBox<T>>>>,
}
impl<T: Default + Reset + Traceable> GuardInner<T> {
fn new() -> Self {
Self {
roots: RefCell::new(Vec::new()),
}
}
fn with_storage(storage: Vec<NonNull<GcBox<T>>>) -> Self {
Self {
roots: RefCell::new(storage),
}
}
}
pub struct Guard<T: Default + Reset + Traceable> {
space: Weak<RefCell<Space<T>>>,
inner: Rc<GuardInner<T>>,
}
impl<T: Default + Reset + Traceable> Guard<T> {
fn new(space: Weak<RefCell<Space<T>>>, inner: Rc<GuardInner<T>>) -> Self {
Self { space, inner }
}
pub fn alloc(&self) -> Gc<T> {
let space = self.space.upgrade().unwrap_or_else(|| {
#[allow(clippy::panic)]
{
panic!("GC error: Heap dropped while guard is still alive")
}
});
let obj = space.borrow_mut().alloc_internal();
let gc_box = unsafe { obj.ptr.as_ref() };
gc_box.ref_count.set(gc_box.ref_count.get() + 1);
self.inner.roots.borrow_mut().push(obj.ptr);
obj
}
pub fn guard(&self, obj: Gc<T>) {
if let Some(_space) = self.space.upgrade() {
let gc_box = unsafe { obj.ptr.as_ref() };
if !gc_box.pooled.get() {
self.inner.roots.borrow_mut().push(obj.ptr);
}
}
}
pub fn unguard(&self, obj: &Gc<T>) -> bool {
let mut roots = self.inner.roots.borrow_mut();
if let Some(pos) = roots.iter().position(|p| *p == obj.ptr) {
roots.swap_remove(pos);
return true;
}
false
}
pub fn clear(&self) {
self.inner.roots.borrow_mut().clear();
}
pub fn len(&self) -> usize {
self.inner.roots.borrow().len()
}
pub fn is_empty(&self) -> bool {
self.inner.roots.borrow().is_empty()
}
pub fn strong_count(&self) -> usize {
Rc::strong_count(&self.inner)
}
}
impl<T: Default + Reset + Traceable> Drop for Guard<T> {
fn drop(&mut self) {
if let Some(space) = self.space.upgrade() {
if Rc::strong_count(&self.inner) == 1 {
let mut roots = self.inner.roots.borrow_mut();
let storage = mem::take(&mut *roots);
drop(roots);
space.borrow_mut().return_guard_to_pool(storage);
}
}
}
}
#[derive(Debug, Clone)]
pub struct GcStats {
pub total_objects: usize,
pub pooled_objects: usize,
pub live_objects: usize,
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Default, Debug)]
struct TestObj {
value: i32,
refs: Vec<Gc<TestObj>>,
}
impl Reset for TestObj {
fn reset(&mut self) {
self.value = 0;
self.refs.clear();
}
}
impl Traceable for TestObj {
fn trace<F: FnMut(GcPtr<Self>)>(&self, mut visitor: F) {
for r in &self.refs {
visitor(r.copy_ref());
}
}
}
#[test]
fn test_basic_alloc() {
let heap: Heap<TestObj> = Heap::new();
let guard = heap.create_guard();
let obj = guard.alloc();
assert_eq!(obj.borrow().value, 0);
obj.borrow_mut().value = 42;
assert_eq!(obj.borrow().value, 42);
let stats = heap.stats();
assert_eq!(stats.live_objects, 1);
assert_eq!(stats.pooled_objects, 0);
}
#[test]
fn test_guard_drop_collects() {
let heap: Heap<TestObj> = Heap::new();
{
let guard = heap.create_guard();
let _obj = guard.alloc();
assert_eq!(heap.stats().live_objects, 1);
}
heap.collect(); assert_eq!(heap.stats().live_objects, 0);
assert_eq!(heap.stats().pooled_objects, 1);
}
#[test]
fn test_pool_reuse() {
let heap: Heap<TestObj> = Heap::new();
{
let guard = heap.create_guard();
let obj = guard.alloc();
obj.borrow_mut().value = 42;
}
heap.collect(); assert_eq!(heap.stats().pooled_objects, 1);
let guard = heap.create_guard();
heap.set_gc_threshold(0); let obj = guard.alloc();
assert_eq!(obj.borrow().value, 0);
assert_eq!(heap.stats().pooled_objects, 0);
assert_eq!(heap.stats().total_objects, 1);
}
#[test]
fn test_ownership_keeps_alive() {
let heap: Heap<TestObj> = Heap::new();
let guard1 = heap.create_guard();
let guard2 = heap.create_guard();
let a = guard1.alloc();
let b = guard2.alloc();
a.borrow_mut().value = 1;
b.borrow_mut().value = 2;
a.borrow_mut().refs.push(b.clone());
drop(guard2);
assert_eq!(heap.stats().live_objects, 2);
assert_eq!(b.borrow().value, 2);
}
#[test]
fn test_disown_allows_collection() {
let heap: Heap<TestObj> = Heap::new();
let guard1 = heap.create_guard();
let guard2 = heap.create_guard();
let a = guard1.alloc();
let b = guard2.alloc();
a.borrow_mut().refs.push(b.clone());
drop(guard2);
drop(b);
heap.collect(); assert_eq!(heap.stats().live_objects, 2);
a.borrow_mut().refs.clear();
heap.collect();
assert_eq!(heap.stats().live_objects, 1);
assert_eq!(heap.stats().pooled_objects, 1);
}
#[test]
fn test_multiple_owners() {
let heap: Heap<TestObj> = Heap::new();
let guard1 = heap.create_guard();
let guard2 = heap.create_guard();
let guard3 = heap.create_guard();
let a = guard1.alloc();
let b = guard2.alloc();
let c = guard3.alloc();
a.borrow_mut().refs.push(c.clone());
b.borrow_mut().refs.push(c.clone());
drop(guard3);
drop(c); heap.collect();
assert_eq!(heap.stats().live_objects, 3);
a.borrow_mut().refs.clear();
heap.collect();
assert_eq!(heap.stats().live_objects, 3);
b.borrow_mut().refs.clear();
heap.collect();
assert_eq!(heap.stats().live_objects, 2); }
#[test]
fn test_cycle_collection() {
let heap: Heap<TestObj> = Heap::new();
let guard = heap.create_guard();
let a = guard.alloc();
let b = guard.alloc();
a.borrow_mut().value = 1;
b.borrow_mut().value = 2;
a.borrow_mut().refs.push(b.clone());
b.borrow_mut().refs.push(a.clone());
assert_eq!(heap.stats().live_objects, 2);
guard.unguard(&a);
heap.collect();
assert_eq!(heap.stats().live_objects, 2);
guard.unguard(&b);
drop(a);
drop(b);
heap.collect();
assert_eq!(heap.stats().live_objects, 0);
}
#[test]
fn test_guard_add_propagates() {
let heap: Heap<TestObj> = Heap::new();
let guard1 = heap.create_guard();
let guard2 = heap.create_guard();
let a = guard1.alloc();
let b = guard1.alloc();
a.borrow_mut().refs.push(b.clone());
guard2.guard(a.clone());
drop(guard1);
heap.collect();
assert_eq!(heap.stats().live_objects, 2);
}
#[test]
fn test_transitive_ownership() {
let heap: Heap<TestObj> = Heap::new();
let guard = heap.create_guard();
let a = guard.alloc();
let b = guard.alloc();
let c = guard.alloc();
a.borrow_mut().refs.push(b.clone());
b.borrow_mut().refs.push(c.clone());
guard.unguard(&b);
guard.unguard(&c);
heap.collect();
assert_eq!(heap.stats().live_objects, 3);
guard.unguard(&a);
drop(a);
drop(b);
drop(c);
heap.collect();
assert_eq!(heap.stats().live_objects, 0);
}
#[test]
fn test_diamond_ownership() {
let heap: Heap<TestObj> = Heap::new();
let guard = heap.create_guard();
let a = guard.alloc();
let b = guard.alloc();
let c = guard.alloc();
let d = guard.alloc();
a.borrow_mut().refs.push(b.clone());
a.borrow_mut().refs.push(c.clone());
b.borrow_mut().refs.push(d.clone());
c.borrow_mut().refs.push(d.clone());
guard.unguard(&b);
guard.unguard(&c);
guard.unguard(&d);
heap.collect();
assert_eq!(heap.stats().live_objects, 4);
b.borrow_mut().refs.retain(|r| !Gc::ptr_eq(r, &d));
heap.collect();
assert_eq!(heap.stats().live_objects, 4);
c.borrow_mut().refs.retain(|r| !Gc::ptr_eq(r, &d));
drop(d); heap.collect();
assert_eq!(heap.stats().live_objects, 3);
}
#[test]
fn test_multiple_references_same_object() {
let heap: Heap<TestObj> = Heap::new();
let guard = heap.create_guard();
let obj1 = guard.alloc();
let obj2 = obj1.clone();
assert_eq!(heap.stats().live_objects, 1);
drop(obj1);
heap.collect();
assert_eq!(heap.stats().live_objects, 1);
drop(obj2);
heap.collect();
assert_eq!(heap.stats().live_objects, 1);
guard.clear();
heap.collect();
assert_eq!(heap.stats().live_objects, 0); }
#[test]
fn test_deep_chain_no_stack_overflow() {
let heap: Heap<TestObj> = Heap::new();
heap.set_gc_threshold(0); let guard = heap.create_guard();
let mut prev = guard.alloc();
for _ in 0..10000 {
let next = guard.alloc();
prev.borrow_mut().refs.push(next.clone());
prev = next;
}
drop(guard);
drop(prev); heap.collect();
assert_eq!(heap.stats().live_objects, 0);
}
}