1#![allow(clippy::missing_safety_doc)]
4#![allow(private_interfaces)]
5
6use std::ptr::NonNull;
7
8pub mod governor;
9pub mod mutator;
10pub mod region;
11pub mod stats;
12
13#[cfg(not(feature = "no-gc"))]
14pub mod cancellation;
15#[cfg(not(feature = "no-gc"))]
16pub mod config;
17
18#[cfg(feature = "no-gc")]
19pub mod alloc_ctx;
20#[cfg(feature = "no-gc")]
21pub mod static_arena;
22
23pub use governor::{
24 Admission, CriticalSection, IsolateAccount, IsolateControl, MemoryCharge, MemoryClass,
25 MemoryConfig, MemoryConfigError, MemoryLimitExceeded, MemorySnapshot, PressureLevel,
26 ProcessMemoryGovernor, ShedHookId, enter_critical, governor, register_current_isolate,
27 resolve_runtime_fault, with_critical,
28};
29pub use stats::{
30 CLJRS_GC_STATS_ENV, GC_STATS, GcStats, GcStatsSnapshot, dump_stats_from_env,
31 report as stats_report,
32};
33
34#[cfg(not(feature = "no-gc"))]
37pub use cancellation::{
38 MutatorGuard, StwGuard, begin_stw, check_cancellation, gc_requested, park_thread,
39 register_mutator, registered_threads, request_gc, safepoint, take_gc_request, unpark_thread,
40 wait_for_threads_to_park,
41};
42#[cfg(not(feature = "no-gc"))]
43pub use config::{GC_CANCELLATION as CONFIG_CANCELLATION, GcConfig, GcParked};
44
45#[cfg(not(feature = "no-gc"))]
46pub use gc_full::{
47 AllocRootGuard, GcHeap, HEAP, HeapProxy, push_alloc_frame, trace_thread_alloc_roots,
48};
49#[cfg(feature = "no-gc")]
50pub use nogc_stubs::{
51 AllocRootGuard, CONFIG_CANCELLATION, GcConfig, GcHeap, GcParked, HEAP, MutatorGuard, StwGuard,
52 begin_stw, check_cancellation, gc_requested, park_thread, push_alloc_frame, register_mutator,
53 registered_threads, request_gc, safepoint, take_gc_request, unpark_thread,
54 wait_for_threads_to_park,
55};
56
57#[cfg(all(feature = "no-gc", debug_assertions))]
62pub fn is_static_addr(addr: usize) -> bool {
63 static_arena::is_static_addr(addr)
64}
65
66pub trait Trace {
80 fn trace(&self, visitor: &mut MarkVisitor);
81
82 fn gc_size_extra(&self) -> usize {
83 0
84 }
85}
86
87impl Trace for String {
90 fn trace(&self, _: &mut MarkVisitor) {}
91
92 fn gc_size_extra(&self) -> usize {
93 self.capacity()
94 }
95}
96impl Trace for i64 {
97 fn trace(&self, _: &mut MarkVisitor) {}
98}
99impl Trace for f64 {
100 fn trace(&self, _: &mut MarkVisitor) {}
101}
102impl Trace for bool {
103 fn trace(&self, _: &mut MarkVisitor) {}
104}
105impl Trace for num_bigint::BigInt {
106 fn trace(&self, _: &mut MarkVisitor) {}
107}
108impl Trace for bigdecimal::BigDecimal {
109 fn trace(&self, _: &mut MarkVisitor) {}
110}
111impl Trace for num_rational::Ratio<num_bigint::BigInt> {
112 fn trace(&self, _: &mut MarkVisitor) {}
113}
114macro_rules! impl_trace_prim_array {
115 ($t:ty) => {
116 impl Trace for std::sync::Mutex<Vec<$t>> {
117 fn trace(&self, _: &mut MarkVisitor) {}
118 fn gc_size_extra(&self) -> usize {
119 self.lock().unwrap().capacity() * std::mem::size_of::<$t>()
120 }
121 }
122 };
123}
124impl_trace_prim_array!(i32);
125impl_trace_prim_array!(i64);
126impl_trace_prim_array!(i16);
127impl_trace_prim_array!(i8);
128impl_trace_prim_array!(char);
129impl_trace_prim_array!(f64);
130impl_trace_prim_array!(f32);
131impl_trace_prim_array!(bool);
132
133pub trait GcVisitor {
136 fn visit<T: Trace + 'static>(&mut self, ptr: &GcPtr<T>);
137}
138
139#[cfg(not(feature = "no-gc"))]
144pub use self::gc_header::{GcBox, GcBoxHeader};
145
146#[cfg(not(feature = "no-gc"))]
147mod gc_header {
148 use crate::{MarkVisitor, Trace};
149 use std::cell::Cell;
150
151 pub(crate) const GC_INITIAL_LIVES: u8 = 2;
159
160 #[cfg(debug_assertions)]
161 pub(crate) const GC_MAGIC_ALIVE: u64 = 0xCAFE_BABE_DEAD_BEEF;
162 #[cfg(debug_assertions)]
163 pub(crate) const GC_MAGIC_FREED: u64 = 0xDEAD_DEAD_DEAD_DEAD;
164
165 #[repr(C)]
166 pub struct GcBoxHeader {
167 #[cfg(debug_assertions)]
168 pub(crate) magic: Cell<u64>,
169 pub(crate) size: usize,
171 pub(crate) lives: Cell<u8>,
172 pub(crate) next: Cell<*mut GcBoxHeader>,
173 pub(crate) trace_fn: unsafe fn(*const GcBoxHeader, &mut MarkVisitor),
174 pub(crate) drop_fn: unsafe fn(*mut GcBoxHeader),
175 }
176
177 impl GcBoxHeader {
178 pub(crate) fn new<T: Trace + 'static>(heap_extra: usize) -> Self {
179 Self {
180 #[cfg(debug_assertions)]
181 magic: Cell::new(GC_MAGIC_ALIVE),
182 size: std::mem::size_of::<GcBox<T>>() + heap_extra,
183 lives: Cell::new(GC_INITIAL_LIVES - 1),
184 next: Cell::new(std::ptr::null_mut()),
185 trace_fn: trace_gc_box::<T>,
186 drop_fn: drop_gc_box::<T>,
187 }
188 }
189 }
190
191 unsafe impl Send for GcBoxHeader {}
192 unsafe impl Sync for GcBoxHeader {}
193
194 #[repr(C)]
195 pub struct GcBox<T: Trace + 'static> {
196 pub(crate) header: GcBoxHeader,
197 pub value: T,
198 }
199
200 pub(crate) unsafe fn trace_gc_box<T: Trace + 'static>(
201 header: *const GcBoxHeader,
202 visitor: &mut MarkVisitor,
203 ) {
204 unsafe {
205 let gc_box = header as *const GcBox<T>;
206 (*gc_box).value.trace(visitor);
207 }
208 }
209
210 pub(crate) unsafe fn drop_gc_box<T: Trace + 'static>(header: *mut GcBoxHeader) {
211 unsafe {
212 #[cfg(debug_assertions)]
213 {
214 (*header).magic.set(GC_MAGIC_FREED);
215 }
216 let gc_box = header as *mut GcBox<T>;
217 drop(Box::from_raw(gc_box));
218 }
219 }
220}
221
222#[cfg(feature = "no-gc")]
227pub use self::nogc_box::GcBox;
228
229#[cfg(feature = "no-gc")]
230mod nogc_box {
231 use crate::Trace;
232
233 pub struct GcBox<T: Trace + 'static> {
234 pub value: T,
235 }
236}
237
238#[cfg(not(feature = "no-gc"))]
243#[derive(Default)]
244pub struct MarkVisitor {
245 pub(crate) grey: Vec<*mut GcBoxHeader>,
246}
247
248#[cfg(feature = "no-gc")]
249pub struct MarkVisitor;
250
251#[cfg(not(feature = "no-gc"))]
252impl MarkVisitor {
253 pub fn new() -> Self {
254 Self::default()
255 }
256
257 pub fn grey_len(&self) -> usize {
258 self.grey.len()
259 }
260
261 pub unsafe fn mark_header(&mut self, header: *mut GcBoxHeader) {
262 use gc_header::GC_INITIAL_LIVES;
263 let h = unsafe { &*header };
264 if h.lives.get() < GC_INITIAL_LIVES {
265 h.lives.set(GC_INITIAL_LIVES);
266 self.grey.push(header);
267 }
268 }
269
270 pub(crate) fn drain(&mut self) {
271 let mut visited = 0usize;
272 while let Some(header) = self.grey.pop() {
273 visited += 1;
274 let h = unsafe { &*header };
275 unsafe { (h.trace_fn)(header as *const GcBoxHeader, self) };
276 }
277 tracing::debug!(target: "gc", "drain visited {} objects", visited);
278 }
279}
280
281#[cfg(not(feature = "no-gc"))]
282impl GcVisitor for MarkVisitor {
283 fn visit<T: Trace + 'static>(&mut self, ptr: &GcPtr<T>) {
284 use gc_header::GC_INITIAL_LIVES;
285 if ptr.is_region_alloc() {
293 return;
294 }
295 let raw = ptr.raw();
296 let header = unsafe { &(*raw).header };
297 if header.lives.get() < GC_INITIAL_LIVES {
298 header.lives.set(GC_INITIAL_LIVES);
299 self.grey.push(raw as *mut GcBoxHeader);
300 }
301 }
302}
303
304#[cfg(feature = "no-gc")]
305impl MarkVisitor {
306 pub fn grey_len(&self) -> usize {
307 0
308 }
309 pub unsafe fn mark_header(&mut self, _: *mut u8) {}
310}
311
312#[cfg(feature = "no-gc")]
313impl GcVisitor for MarkVisitor {
314 fn visit<T: Trace + 'static>(&mut self, _: &GcPtr<T>) {}
315}
316
317pub struct GcPtr<T: Trace + 'static>(NonNull<GcBox<T>>);
322
323#[cfg(not(feature = "no-gc"))]
332pub(crate) const REGION_PTR_TAG: usize = 1;
333
334pub struct AllocError<T> {
337 value: T,
338 error: MemoryLimitExceeded,
339}
340
341impl<T> AllocError<T> {
342 pub fn error(&self) -> MemoryLimitExceeded {
344 self.error
345 }
346
347 pub fn into_inner(self) -> T {
349 self.value
350 }
351
352 pub fn into_parts(self) -> (T, MemoryLimitExceeded) {
354 (self.value, self.error)
355 }
356}
357
358impl<T> std::fmt::Debug for AllocError<T> {
359 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
360 f.debug_struct("AllocError")
361 .field("error", &self.error)
362 .finish_non_exhaustive()
363 }
364}
365
366impl<T> std::fmt::Display for AllocError<T> {
367 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
368 self.error.fmt(f)
369 }
370}
371
372impl<T> std::error::Error for AllocError<T> {}
373
374impl<T> From<AllocError<T>> for MemoryLimitExceeded {
375 fn from(e: AllocError<T>) -> Self {
376 e.error
377 }
378}
379
380impl<T: Trace + 'static> GcPtr<T> {
381 #[cfg(not(feature = "no-gc"))]
386 pub fn new(value: T) -> Self {
387 gc_full::HEAP.alloc(value)
388 }
389
390 #[cfg(feature = "no-gc")]
391 pub fn new(value: T) -> Self {
392 alloc_ctx::alloc_in_ctx(value)
393 }
394
395 pub fn try_new(value: T) -> Result<Self, AllocError<T>> {
404 #[cfg(not(feature = "no-gc"))]
405 {
406 let bytes = std::mem::size_of::<GcBox<T>>() + value.gc_size_extra();
407 match governor::try_reserve_current(bytes) {
408 Ok(()) => Ok(Self::new(value)),
409 Err(error) => Err(AllocError { value, error }),
410 }
411 }
412 #[cfg(feature = "no-gc")]
413 {
414 alloc_ctx::try_alloc_in_ctx(value)
415 }
416 }
417
418 pub fn new_critical(value: T) -> Self {
422 let _section = governor::enter_critical();
423 Self::new(value)
424 }
425
426 #[inline]
429 fn raw(&self) -> *mut GcBox<T> {
430 #[cfg(not(feature = "no-gc"))]
431 {
432 (self.0.as_ptr() as usize & !REGION_PTR_TAG) as *mut GcBox<T>
433 }
434 #[cfg(feature = "no-gc")]
435 {
436 self.0.as_ptr()
437 }
438 }
439
440 #[cfg(not(feature = "no-gc"))]
443 #[inline]
444 pub fn is_region_alloc(&self) -> bool {
445 (self.0.as_ptr() as usize & REGION_PTR_TAG) != 0
446 }
447
448 #[cfg(not(feature = "no-gc"))]
455 #[inline]
456 pub(crate) unsafe fn from_region_raw(raw: *mut GcBox<T>) -> Self {
457 GcPtr(unsafe { NonNull::new_unchecked((raw as usize | REGION_PTR_TAG) as *mut GcBox<T>) })
458 }
459
460 pub fn get(&self) -> &T {
461 #[cfg(all(debug_assertions, not(feature = "no-gc")))]
462 {
463 use gc_header::GC_MAGIC_ALIVE;
464 let header = unsafe { &(*self.raw()).header };
465 assert_eq!(
466 header.magic.get(),
467 GC_MAGIC_ALIVE,
468 "GcPtr::get() on freed object! magic={:#x}",
469 header.magic.get(),
470 );
471 }
472 unsafe { &(*self.raw()).value }
473 }
474
475 pub fn get_mut(&mut self) -> &mut T {
476 #[cfg(all(debug_assertions, not(feature = "no-gc")))]
477 {
478 use gc_header::GC_MAGIC_ALIVE;
479 let header = unsafe { &(*self.raw()).header };
480 assert_eq!(
481 header.magic.get(),
482 GC_MAGIC_ALIVE,
483 "GcPtr::get_mut() on freed object! magic={:#x}",
484 header.magic.get(),
485 );
486 }
487 unsafe { &mut (*self.raw()).value }
488 }
489
490 pub fn ptr_eq(a: &Self, b: &Self) -> bool {
491 a.0 == b.0
492 }
493
494 #[cfg(all(feature = "no-gc", debug_assertions))]
500 pub fn is_static_alloc(&self) -> bool {
501 alloc_ctx::invocation_is_active() || static_arena::is_static_addr(self.0.as_ptr() as usize)
502 }
503}
504
505impl<T: Trace + 'static> Clone for GcPtr<T> {
506 fn clone(&self) -> Self {
507 GcPtr(self.0)
508 }
509}
510
511impl<T: Trace + 'static + std::fmt::Debug> std::fmt::Debug for GcPtr<T> {
512 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
513 unsafe { (*self.raw()).value.fmt(f) }
514 }
515}
516
517impl<T: Trace + 'static> Drop for GcPtr<T> {
518 fn drop(&mut self) {}
519}
520
521pub struct StaticGcPtr<T: 'static>(NonNull<T>);
535
536unsafe impl<T: 'static> Send for StaticGcPtr<T> {}
540unsafe impl<T: 'static> Sync for StaticGcPtr<T> {}
541
542impl<T: 'static> StaticGcPtr<T> {
543 pub fn get(&self) -> &T {
545 unsafe { self.0.as_ref() }
547 }
548
549 pub fn ptr_eq(a: &Self, b: &Self) -> bool {
552 a.0 == b.0
553 }
554}
555
556impl<T: 'static> Clone for StaticGcPtr<T> {
557 fn clone(&self) -> Self {
558 StaticGcPtr(self.0)
559 }
560}
561
562impl<T: 'static + std::fmt::Debug> std::fmt::Debug for StaticGcPtr<T> {
563 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
564 unsafe { self.0.as_ref().fmt(f) }
565 }
566}
567
568pub fn static_alloc<T: 'static>(value: T) -> StaticGcPtr<T> {
576 static_alloc_inner(value)
577}
578
579pub fn try_static_alloc<T: 'static>(value: T) -> Result<StaticGcPtr<T>, AllocError<T>> {
583 #[cfg(not(feature = "no-gc"))]
584 {
585 match governor::governor()
586 .try_charge_permanent(governor::MemoryClass::Static, std::mem::size_of::<T>())
587 {
588 Ok(()) => Ok(StaticGcPtr(NonNull::from(Box::leak(Box::new(value))))),
589 Err(error) => Err(AllocError { value, error }),
590 }
591 }
592 #[cfg(feature = "no-gc")]
593 {
594 static_arena::static_arena().try_alloc_val(value)
595 }
596}
597
598fn static_alloc_inner<T: 'static>(value: T) -> StaticGcPtr<T> {
599 #[cfg(feature = "no-gc")]
600 {
601 static_arena::static_alloc_val(value)
602 }
603 #[cfg(not(feature = "no-gc"))]
604 {
605 governor::governor()
606 .charge_permanent(governor::MemoryClass::Static, std::mem::size_of::<T>());
607 StaticGcPtr(NonNull::from(Box::leak(Box::new(value))))
608 }
609}
610
611#[cfg(not(feature = "no-gc"))]
616mod gc_full {
617 use std::cell::RefCell;
618 use std::ptr::NonNull;
619 use std::sync::atomic::{AtomicUsize, Ordering};
620 use std::sync::{Arc, Mutex};
621
622 use crate::config::GcConfig;
623 use crate::gc_header::GC_INITIAL_LIVES;
624 use crate::{GcBox, GcBoxHeader, GcPtr, MarkVisitor, Trace};
625
626 type RootTracer = Box<dyn Fn(&mut MarkVisitor)>;
627
628 pub struct GcHeap {
629 inner: Mutex<GcHeapInner>,
630 config: Mutex<Option<Arc<GcConfig>>>,
631 memory_in_use: AtomicUsize,
632 total_allocated_bytes: AtomicUsize,
633 root_tracers: Mutex<Vec<RootTracer>>,
634 gc_suppressed: std::sync::atomic::AtomicBool,
635 suppressed_threshold: AtomicUsize,
642 zero_yield_headroom: AtomicUsize,
644 }
645
646 struct GcHeapInner {
647 head: *mut GcBoxHeader,
648 count: usize,
649 total_allocated: usize,
650 total_freed: usize,
651 }
652
653 unsafe impl Send for GcHeapInner {}
654
655 impl GcHeapInner {
656 const fn new() -> Self {
657 Self {
658 head: std::ptr::null_mut(),
659 count: 0,
660 total_allocated: 0,
661 total_freed: 0,
662 }
663 }
664 }
665
666 unsafe impl Sync for GcHeap {}
667
668 impl Default for GcHeap {
669 fn default() -> Self {
670 Self::new()
671 }
672 }
673
674 pub(crate) fn parse_limit_mb(var: &str, value: Option<&str>, default: usize) -> usize {
679 match value {
680 Some(s) => match s.trim().parse::<usize>() {
681 Ok(mb) => mb.saturating_mul(1024 * 1024),
682 Err(_) => {
683 eprintln!(
684 "[gc] warning: ignoring invalid {var}={s:?} (expected a number of megabytes)"
685 );
686 default
687 }
688 },
689 None => default,
690 }
691 }
692
693 impl GcHeap {
694 pub const fn new() -> Self {
695 Self {
696 inner: Mutex::new(GcHeapInner::new()),
697 config: Mutex::new(None),
698 memory_in_use: AtomicUsize::new(0),
699 total_allocated_bytes: AtomicUsize::new(0),
700 root_tracers: Mutex::new(Vec::new()),
701 gc_suppressed: std::sync::atomic::AtomicBool::new(false),
702 suppressed_threshold: AtomicUsize::new(0),
703 zero_yield_headroom: AtomicUsize::new(0),
704 }
705 }
706
707 pub fn set_config(&self, config: Arc<GcConfig>) {
708 *self.config.lock().unwrap() = Some(config);
709 }
710
711 pub fn set_config_from_env(&self) {
712 #[cfg(not(target_arch = "wasm32"))]
713 let default_soft_limit: usize = (system_memory::total() / 3) as usize;
714 #[cfg(target_arch = "wasm32")]
715 let default_soft_limit: usize = 64 * 1024 * 1024;
716
717 let soft_limit_mb = parse_limit_mb(
718 "CLJRS_GC_SOFT_LIMIT_MB",
719 std::env::var("CLJRS_GC_SOFT_LIMIT_MB").ok().as_deref(),
720 default_soft_limit,
721 );
722 let mut hard_limit_mb = parse_limit_mb(
723 "CLJRS_GC_HARD_LIMIT_MB",
724 std::env::var("CLJRS_GC_HARD_LIMIT_MB").ok().as_deref(),
725 soft_limit_mb,
726 );
727 if hard_limit_mb < soft_limit_mb {
728 eprintln!(
729 "[gc] warning: CLJRS_GC_HARD_LIMIT_MB is below CLJRS_GC_SOFT_LIMIT_MB; using the soft limit for both"
730 );
731 hard_limit_mb = soft_limit_mb;
732 }
733 self.set_config(Arc::new(GcConfig::with_limits(
734 soft_limit_mb,
735 hard_limit_mb,
736 )));
737 }
738
739 pub fn register_root_tracer(&self, tracer: impl Fn(&mut MarkVisitor) + 'static) {
740 self.root_tracers.lock().unwrap().push(Box::new(tracer));
741 }
742
743 pub fn trace_registered_roots(&self, visitor: &mut MarkVisitor) {
744 let tracers = self.root_tracers.lock().unwrap();
745 for tracer in tracers.iter() {
746 tracer(visitor);
747 }
748 }
749
750 pub fn memory_in_use(&self) -> usize {
751 self.memory_in_use.load(Ordering::Relaxed)
752 }
753
754 #[cfg(test)]
755 pub fn set_memory_in_use(&self, bytes: usize) {
756 self.memory_in_use.store(bytes, Ordering::Relaxed);
757 }
758
759 pub fn alloc<T: Trace + 'static>(&self, value: T) -> GcPtr<T> {
760 crate::cancellation::safepoint();
761 let heap_extra = value.gc_size_extra();
762 let gc_box = Box::new(GcBox {
763 header: GcBoxHeader::new::<T>(heap_extra),
764 value,
765 });
766 let obj_size = gc_box.header.size; let raw: *mut GcBox<T> = Box::into_raw(gc_box);
768 {
769 let mut inner = self.inner.lock().unwrap();
770 unsafe {
771 (*raw).header.next.set(inner.head);
772 inner.head = raw as *mut GcBoxHeader;
773 }
774 inner.count += 1;
775 inner.total_allocated += 1;
776 }
777 self.total_allocated_bytes
778 .fetch_add(obj_size, Ordering::Relaxed);
779 if crate::governor::with_current_account(|a| a.charge(obj_size)).is_none() {
782 crate::stats::GC_STATS.record_gc_alloc(obj_size);
783 }
784 let current_usage =
785 self.memory_in_use.fetch_add(obj_size, Ordering::Relaxed) + obj_size;
786
787 if let Some(config) = self.config.lock().unwrap().as_ref()
788 && config.soft_limit_exceeded(current_usage)
789 {
790 if self.gc_suppressed.load(Ordering::Relaxed) {
791 let threshold = self.suppressed_threshold.load(Ordering::Relaxed);
795 if current_usage > threshold {
796 self.gc_suppressed.store(false, Ordering::Relaxed);
797 crate::cancellation::request_gc();
798 }
799 } else {
800 crate::cancellation::request_gc();
801 }
802 }
803
804 register_alloc(raw as *mut GcBoxHeader);
805 GcPtr(unsafe { NonNull::new_unchecked(raw) })
806 }
807
808 pub fn collect<F: FnOnce(&mut MarkVisitor)>(&self, trace_roots: F) {
809 let pre_count = self.inner.lock().unwrap().count;
810 let pre_memory = self.memory_in_use.load(Ordering::Relaxed);
811 tracing::debug!(
812 target: "gc",
813 "starting collection: {} objects, ~{} bytes in use",
814 pre_count,
815 pre_memory
816 );
817 let mark_start = std::time::Instant::now();
818 let mut visitor = MarkVisitor::new();
819 trace_roots(&mut visitor);
820 crate::region::trace_active_regions(&mut visitor);
826 crate::region::trace_retired_regions(&mut visitor);
829 tracing::debug!(
830 target: "gc",
831 "starting drain with {} grey objects",
832 visitor.grey.len()
833 );
834 visitor.drain();
835 let mark_elapsed = mark_start.elapsed();
836
837 let sweep_start = std::time::Instant::now();
838 let mut inner = self.inner.lock().unwrap();
839 let mut live: Vec<*mut GcBoxHeader> = Vec::with_capacity(inner.count);
840 let mut dead: Vec<*mut GcBoxHeader> = Vec::new();
841 let mut freed_bytes: usize = 0;
843 let mut pending_bytes: usize = 0;
846 let mut current = inner.head;
847 while !current.is_null() {
848 let header = unsafe { &*current };
849 let next = header.next.get();
850 let lives = header.lives.get();
851 let obj_size = header.size;
852 if lives >= GC_INITIAL_LIVES {
853 header.lives.set(GC_INITIAL_LIVES - 1);
855 live.push(current);
856 } else if lives > 0 {
857 header.lives.set(lives - 1);
859 if lives == 1 {
860 pending_bytes += obj_size;
861 }
862 live.push(current);
863 } else {
864 freed_bytes += obj_size;
866 dead.push(current);
867 }
868 current = next;
869 }
870 let freed_count = dead.len();
871 for ptr in dead {
872 let header = unsafe { &*ptr };
873 unsafe { (header.drop_fn)(ptr) };
874 inner.count -= 1;
875 inner.total_freed += 1;
876 }
877 inner.head = std::ptr::null_mut();
878 for ptr in live {
879 let header = unsafe { &*ptr };
880 header.next.set(inner.head);
881 inner.head = ptr;
882 }
883 self.memory_in_use.fetch_sub(freed_bytes, Ordering::Relaxed);
888 let sweep_elapsed = sweep_start.elapsed();
889 let post_memory = self.memory_in_use.load(Ordering::Relaxed);
890 crate::governor::with_current_account(|a| {
891 a.release(freed_bytes);
892 a.record_collection(crate::governor::CollectionReport {
893 bytes_before: pre_memory,
894 bytes_after: post_memory,
895 bytes_returned: freed_bytes,
896 bytes_pending: pending_bytes,
897 duration: mark_elapsed + sweep_elapsed,
898 });
899 });
900 crate::stats::GC_STATS.record_gc_pause(
901 mark_elapsed + sweep_elapsed,
902 freed_count as u64,
903 freed_bytes as u64,
904 );
905 tracing::debug!(
906 target: "gc",
907 "collection complete: freed {} (~{} bytes), {} remaining (~{} bytes, ~{} pending), mark={:.2?} sweep={:.2?}",
908 freed_count,
909 freed_bytes,
910 inner.count,
911 post_memory,
912 pending_bytes,
913 mark_elapsed,
914 sweep_elapsed
915 );
916 if freed_count == 0 && pending_bytes == 0 {
917 let soft_limit = self
932 .config
933 .lock()
934 .unwrap()
935 .as_ref()
936 .map(|c| c.soft_limit())
937 .unwrap_or(64 * 1024 * 1024);
938 let base_headroom = (soft_limit / 10).max(1);
939 let max_headroom = (soft_limit / 4).max(base_headroom);
940 let prev_headroom = self.zero_yield_headroom.load(Ordering::Relaxed);
941 let headroom = if prev_headroom == 0 {
942 base_headroom
943 } else {
944 prev_headroom.saturating_mul(2).min(max_headroom)
945 };
946 self.zero_yield_headroom.store(headroom, Ordering::Relaxed);
947 self.suppressed_threshold
948 .store(post_memory + headroom, Ordering::Relaxed);
949 self.gc_suppressed.store(true, Ordering::Relaxed);
950 } else {
951 self.zero_yield_headroom.store(0, Ordering::Relaxed);
953 self.gc_suppressed.store(false, Ordering::Relaxed);
954 }
955 }
956
957 pub fn count(&self) -> usize {
958 self.inner.lock().unwrap().count
959 }
960 pub fn total_allocated(&self) -> usize {
961 self.inner.lock().unwrap().total_allocated
962 }
963 pub fn total_freed(&self) -> usize {
964 self.inner.lock().unwrap().total_freed
965 }
966
967 pub fn collect_auto(&self) -> bool {
968 tracing::debug!(target: "gc", "automatic collection requested");
969 let Some(_stw_guard) = crate::cancellation::begin_stw() else {
970 tracing::debug!(target: "gc", "automatic collection skipped");
971 return false;
972 };
973 self.collect(|visitor| self.trace_registered_roots(visitor));
974 true
975 }
976 }
977
978 thread_local! {
979 static ISOLATE_HEAP: GcHeap = const { GcHeap::new() };
980 }
981
982 pub struct HeapProxy;
988
989 impl HeapProxy {
990 pub fn alloc<T: Trace + 'static>(&self, value: T) -> GcPtr<T> {
991 ISOLATE_HEAP.with(|h| h.alloc(value))
992 }
993
994 pub fn set_config(&self, config: Arc<GcConfig>) {
995 ISOLATE_HEAP.with(|h| h.set_config(config));
996 }
997
998 pub fn set_config_from_env(&self) {
999 ISOLATE_HEAP.with(|h| h.set_config_from_env());
1000 }
1001
1002 pub fn register_root_tracer(&self, tracer: impl Fn(&mut MarkVisitor) + 'static) {
1003 ISOLATE_HEAP.with(|h| h.register_root_tracer(tracer));
1004 }
1005
1006 pub fn trace_registered_roots(&self, visitor: &mut MarkVisitor) {
1007 ISOLATE_HEAP.with(|h| h.trace_registered_roots(visitor));
1008 }
1009
1010 pub fn memory_in_use(&self) -> usize {
1011 ISOLATE_HEAP.with(|h| h.memory_in_use())
1012 }
1013
1014 pub fn count(&self) -> usize {
1015 ISOLATE_HEAP.with(|h| h.count())
1016 }
1017
1018 pub fn total_allocated(&self) -> usize {
1019 ISOLATE_HEAP.with(|h| h.total_allocated())
1020 }
1021
1022 pub fn total_freed(&self) -> usize {
1023 ISOLATE_HEAP.with(|h| h.total_freed())
1024 }
1025
1026 pub fn collect<F: FnOnce(&mut MarkVisitor)>(&self, trace_roots: F) {
1027 ISOLATE_HEAP.with(|h| h.collect(trace_roots));
1028 }
1029
1030 pub fn collect_auto(&self) -> bool {
1031 ISOLATE_HEAP.with(|h| h.collect_auto())
1032 }
1033
1034 #[cfg(test)]
1035 pub fn set_memory_in_use(&self, bytes: usize) {
1036 ISOLATE_HEAP.with(|h| h.set_memory_in_use(bytes));
1037 }
1038 }
1039
1040 unsafe impl Sync for HeapProxy {}
1043
1044 pub static HEAP: HeapProxy = HeapProxy;
1045
1046 thread_local! {
1047 pub(crate) static ALLOC_ROOTS: RefCell<Vec<*mut GcBoxHeader>> = const { RefCell::new(Vec::new()) };
1048 }
1049 crate::mutator_slot!("cljrs-gc/ALLOC_ROOTS", refcell ALLOC_ROOTS: Vec<*mut GcBoxHeader>);
1050
1051 pub struct AllocRootGuard {
1052 saved_len: usize,
1053 }
1054
1055 impl AllocRootGuard {
1056 pub fn release(&self) {
1060 ALLOC_ROOTS.with(|roots| roots.borrow_mut().truncate(self.saved_len));
1061 }
1062 }
1063
1064 impl Drop for AllocRootGuard {
1065 fn drop(&mut self) {
1066 self.release();
1067 }
1068 }
1069
1070 pub fn push_alloc_frame() -> AllocRootGuard {
1071 let saved_len = ALLOC_ROOTS.with(|roots| roots.borrow().len());
1072 AllocRootGuard { saved_len }
1073 }
1074
1075 fn register_alloc(header: *mut GcBoxHeader) {
1076 ALLOC_ROOTS.with(|roots| roots.borrow_mut().push(header));
1077 }
1078
1079 pub fn trace_thread_alloc_roots(visitor: &mut MarkVisitor) {
1080 ALLOC_ROOTS.with(|roots| {
1081 let roots = roots.borrow();
1082 for &header in roots.iter() {
1083 unsafe { visitor.mark_header(header) };
1084 }
1085 });
1086 }
1087}
1088
1089#[cfg(feature = "no-gc")]
1094mod nogc_stubs {
1095 use crate::MarkVisitor;
1096 use std::sync::Arc;
1097
1098 #[derive(Debug, Clone)]
1099 pub struct GcConfig;
1100 impl GcConfig {
1101 pub fn new() -> Self {
1102 Self
1103 }
1104 pub fn with_hard_limit(_: usize) -> Self {
1105 Self
1106 }
1107 pub fn with_limits(_: usize, _: usize) -> Self {
1108 Self
1109 }
1110 pub fn with_soft_limit(_: usize) -> Self {
1111 Self
1112 }
1113 pub fn try_with_limits(
1114 soft_limit: usize,
1115 hard_limit: usize,
1116 ) -> Result<Self, crate::governor::MemoryConfigError> {
1117 crate::governor::MemoryConfig::with_limits(soft_limit, hard_limit).map(|_| Self)
1118 }
1119 pub fn validate(&self) -> Result<(), crate::governor::MemoryConfigError> {
1120 Ok(())
1121 }
1122 }
1123 impl Default for GcConfig {
1124 fn default() -> Self {
1125 Self::new()
1126 }
1127 }
1128
1129 pub struct GcHeap;
1130 impl Default for GcHeap {
1131 fn default() -> Self {
1132 Self::new()
1133 }
1134 }
1135 impl GcHeap {
1136 pub const fn new() -> Self {
1137 Self
1138 }
1139 pub fn set_config(&self, _: Arc<GcConfig>) {}
1140 pub fn set_config_from_env(&self) {}
1145 pub fn register_root_tracer(&self, _: impl Fn(&mut MarkVisitor) + 'static) {}
1146 pub fn trace_registered_roots(&self, _: &mut MarkVisitor) {}
1147 pub fn memory_in_use(&self) -> usize {
1148 0
1149 }
1150 pub fn count(&self) -> usize {
1151 0
1152 }
1153 pub fn total_allocated(&self) -> usize {
1154 0
1155 }
1156 pub fn total_freed(&self) -> usize {
1157 0
1158 }
1159 pub fn collect<F: FnOnce(&mut MarkVisitor)>(&self, _: F) {}
1160 pub fn collect_auto(&self) -> bool {
1161 false
1162 }
1163 }
1164 unsafe impl Sync for GcHeap {}
1165 pub static HEAP: GcHeap = GcHeap::new();
1166
1167 pub struct MutatorGuard;
1168 impl Drop for MutatorGuard {
1169 fn drop(&mut self) {}
1170 }
1171 pub struct StwGuard;
1172 impl Drop for StwGuard {
1173 fn drop(&mut self) {}
1174 }
1175 pub struct GcParked;
1176
1177 pub struct GcCancellationStub;
1178 impl GcCancellationStub {
1179 pub const fn new() -> Self {
1180 Self
1181 }
1182 pub fn in_progress(&self) -> bool {
1183 false
1184 }
1185 }
1186 pub static CONFIG_CANCELLATION: GcCancellationStub = GcCancellationStub::new();
1187
1188 pub fn safepoint() {}
1189 pub fn gc_requested() -> bool {
1190 false
1191 }
1192 pub fn take_gc_request() -> bool {
1193 false
1194 }
1195 pub fn begin_stw() -> Option<StwGuard> {
1196 None
1197 }
1198 pub fn register_mutator() -> MutatorGuard {
1199 MutatorGuard
1200 }
1201 pub fn registered_threads() -> usize {
1202 0
1203 }
1204 pub fn request_gc() {}
1205 pub fn check_cancellation() -> Result<(), GcParked> {
1206 Ok(())
1207 }
1208 pub fn park_thread() {}
1209 pub fn unpark_thread() {}
1210 pub fn wait_for_threads_to_park() {}
1211
1212 pub struct AllocRootGuard;
1213 impl AllocRootGuard {
1214 pub fn release(&self) {}
1215 }
1216 impl Drop for AllocRootGuard {
1217 fn drop(&mut self) {}
1218 }
1219 pub fn push_alloc_frame() -> AllocRootGuard {
1220 AllocRootGuard
1221 }
1222}
1223
1224#[cfg(all(test, not(feature = "no-gc")))]
1229mod tests {
1230 use super::*;
1231 use std::sync::{Arc, Mutex};
1232
1233 #[derive(Debug)]
1234 #[allow(dead_code)]
1235 struct Tracked {
1236 value: i32,
1237 dropped: Arc<Mutex<bool>>,
1238 }
1239 impl Drop for Tracked {
1240 fn drop(&mut self) {
1241 *self.dropped.lock().unwrap() = true;
1242 }
1243 }
1244 impl Trace for Tracked {
1245 fn trace(&self, _: &mut MarkVisitor) {}
1246 }
1247
1248 #[derive(Debug)]
1249 #[allow(dead_code)]
1250 struct Parent {
1251 child: GcPtr<Tracked>,
1252 }
1253 impl Trace for Parent {
1254 fn trace(&self, visitor: &mut MarkVisitor) {
1255 visitor.visit(&self.child);
1256 }
1257 }
1258
1259 fn fresh_heap() -> gc_full::GcHeap {
1260 let heap = gc_full::GcHeap::new();
1261 heap.set_config(Arc::new(GcConfig::with_limits(10000, 50000)));
1262 heap
1263 }
1264
1265 #[test]
1266 fn alloc_and_get() {
1267 let heap = fresh_heap();
1268 let p = heap.alloc(42i64);
1269 assert_eq!(*p.get(), 42);
1270 assert_eq!(heap.count(), 1);
1271 }
1272
1273 #[test]
1274 fn clone_is_same_ptr() {
1275 let heap = fresh_heap();
1276 let p = heap.alloc(99i64);
1277 let q = p.clone();
1278 assert!(GcPtr::ptr_eq(&p, &q));
1279 }
1280
1281 #[test]
1282 fn parse_limit_mb_handles_valid_unset_and_malformed() {
1283 assert_eq!(gc_full::parse_limit_mb("X", Some("4"), 7), 4 * 1024 * 1024);
1285 assert_eq!(
1287 gc_full::parse_limit_mb("X", Some(" 4 "), 7),
1288 4 * 1024 * 1024
1289 );
1290 assert_eq!(gc_full::parse_limit_mb("X", None, 7), 7);
1292 assert_eq!(gc_full::parse_limit_mb("X", Some("foo"), 7), 7);
1294 assert_eq!(gc_full::parse_limit_mb("X", Some(""), 7), 7);
1295 assert_eq!(gc_full::parse_limit_mb("X", Some("-1"), 7), 7);
1296 assert_eq!(
1298 gc_full::parse_limit_mb("X", Some(&usize::MAX.to_string()), 7),
1299 usize::MAX
1300 );
1301 }
1302
1303 #[test]
1304 fn collect_keeps_reachable() {
1305 let heap = fresh_heap();
1306 let dropped = Arc::new(Mutex::new(false));
1307 let p = heap.alloc(Tracked {
1308 value: 2,
1309 dropped: dropped.clone(),
1310 });
1311 heap.collect(|vis| vis.visit(&p));
1312 assert_eq!(heap.count(), 1);
1313 assert!(!*dropped.lock().unwrap());
1314 }
1315
1316 #[test]
1317 fn b1_two_isolates_independent_heaps() {
1318 use std::sync::{Arc, Barrier};
1319 let barrier = Arc::new(Barrier::new(2));
1322 let b1 = barrier.clone();
1323 let h1 = std::thread::Builder::new()
1324 .name("isolate-1".into())
1325 .spawn(move || {
1326 let _mutator = crate::register_mutator();
1327 let _ptrs: Vec<_> = (0_i64..100)
1329 .map(|i| crate::gc_full::HEAP.alloc(i))
1330 .collect();
1331 b1.wait(); assert_eq!(
1334 crate::gc_full::HEAP.count(),
1335 100,
1336 "isolate-1 heap count should be 100"
1337 );
1338 })
1339 .unwrap();
1340
1341 let b2 = barrier.clone();
1342 let h2 = std::thread::Builder::new()
1343 .name("isolate-2".into())
1344 .spawn(move || {
1345 let _mutator = crate::register_mutator();
1346 let _ptrs: Vec<_> = (0_i64..200)
1348 .map(|i| crate::gc_full::HEAP.alloc(i))
1349 .collect();
1350 b2.wait();
1351 assert_eq!(
1353 crate::gc_full::HEAP.count(),
1354 200,
1355 "isolate-2 heap count should be 200"
1356 );
1357 })
1358 .unwrap();
1359
1360 h1.join().expect("isolate-1 panicked");
1361 h2.join().expect("isolate-2 panicked");
1362 }
1363
1364 #[test]
1365 fn b1_two_isolates_gc_independently() {
1366 let h1 = std::thread::Builder::new()
1368 .name("gc-isolate-1".into())
1369 .spawn(|| {
1370 let _mutator = crate::register_mutator();
1371 let heap = &crate::gc_full::HEAP;
1372 heap.set_config(Arc::new(GcConfig::with_limits(16_384, 65_536)));
1373 for _ in 0..5 {
1375 let _ptrs: Vec<_> = (0_i64..50).map(|i| heap.alloc(i)).collect();
1376 heap.collect(|_| {});
1378 heap.collect(|_| {}); }
1380 })
1384 .unwrap();
1385
1386 let h2 = std::thread::Builder::new()
1387 .name("gc-isolate-2".into())
1388 .spawn(|| {
1389 let _mutator = crate::register_mutator();
1390 let heap = &crate::gc_full::HEAP;
1391 heap.set_config(Arc::new(GcConfig::with_limits(16_384, 65_536)));
1392 for _ in 0..5 {
1393 let _ptrs: Vec<_> = (0_i64..50).map(|i| heap.alloc(i)).collect();
1394 heap.collect(|_| {});
1395 heap.collect(|_| {});
1396 }
1397 })
1398 .unwrap();
1399
1400 h1.join().expect("gc-isolate-1 panicked");
1401 h2.join().expect("gc-isolate-2 panicked");
1402 }
1403}
1404
1405#[cfg(all(test, feature = "no-gc"))]
1406mod nogc_tests {
1407 use super::*;
1408 use alloc_ctx::{ScratchGuard, StaticCtxGuard};
1409
1410 #[test]
1411 fn alloc_in_static_context() {
1412 let _g = StaticCtxGuard::new();
1413 let p = GcPtr::new(42i64);
1414 assert_eq!(*p.get(), 42);
1415 }
1416
1417 #[test]
1418 fn alloc_in_scratch_region() {
1419 let mut scratch = ScratchGuard::new();
1420 let p = GcPtr::new(99i64);
1421 assert_eq!(*p.get(), 99);
1422 scratch.pop_for_return();
1423 assert_eq!(*p.get(), 99);
1424 }
1426
1427 #[test]
1428 fn ptr_eq() {
1429 let _g = StaticCtxGuard::new();
1430 let p = GcPtr::new(1i64);
1431 let q = p.clone();
1432 assert!(GcPtr::ptr_eq(&p, &q));
1433 }
1434}