use core::cell::Cell;
#[cfg(debug_assertions)]
use core::ffi::c_void;
use core::marker::PhantomData;
use core::ptr::NonNull;
use core::sync::atomic::{AtomicU32, Ordering};
use bun_core::StoredTrace;
use bun_core::ThreadLock;
#[cfg(debug_assertions)]
use std::collections::HashMap;
#[cfg(debug_assertions)]
type ArrayHashMap<K, V> = HashMap<K, V>;
#[inline]
fn dump_stack_hook(trace: Option<&StoredTrace>, ret_addr: usize) {
match trace {
None => bun_core::dump_current_stack_trace(
if ret_addr == 0 { None } else { Some(ret_addr) },
bun_core::DumpStackTraceOptions::default(),
),
Some(stored) => {
bun_core::dump_stack_trace(&stored.trace(), bun_core::DumpStackTraceOptions::default())
}
}
}
#[derive(Default)]
pub struct Options {
pub debug_name: Option<&'static str>,
}
pub trait RefCounted: Sized {
type DestructorCtx;
fn debug_name() -> &'static str {
core::any::type_name::<Self>()
}
unsafe fn get_ref_count(this: *mut Self) -> *mut RefCount<Self>;
unsafe fn destructor(this: *mut Self, ctx: Self::DestructorCtx);
}
pub trait ThreadSafeRefCounted: Sized {
fn debug_name() -> &'static str {
core::any::type_name::<Self>()
}
unsafe fn get_ref_count(this: *mut Self) -> *mut ThreadSafeRefCount<Self>;
#[inline]
unsafe fn destructor(this: *mut Self) {
drop(unsafe { Box::from_raw(this) });
}
}
pub trait AnyRefCounted: Sized {
type DestructorCtx;
unsafe fn rc_ref(this: *mut Self);
unsafe fn rc_deref_with_context(this: *mut Self, ctx: Self::DestructorCtx);
#[inline]
unsafe fn rc_deref(this: *mut Self)
where
Self::DestructorCtx: Default,
{
unsafe { Self::rc_deref_with_context(this, Default::default()) }
}
unsafe fn rc_has_one_ref(this: *const Self) -> bool;
unsafe fn rc_assert_no_refs(this: *const Self);
#[cfg(debug_assertions)]
unsafe fn rc_debug_data(this: *mut Self) -> *mut dyn DebugDataOps;
}
#[inline]
pub fn finalize_js_box<T, F>(boxed: Box<T>, before: F)
where
T: AnyRefCounted,
T::DestructorCtx: Default,
F: FnOnce(&T),
{
let ptr: *mut T = Box::into_raw(boxed);
before(unsafe { &*ptr });
unsafe { T::rc_deref(ptr) };
}
#[inline]
pub fn finalize_js_box_noop<T>(boxed: Box<T>)
where
T: AnyRefCounted,
T::DestructorCtx: Default,
{
let ptr: *mut T = Box::into_raw(boxed);
unsafe { T::rc_deref(ptr) };
}
pub struct RefCount<T: RefCounted> {
raw_count: Cell<u32>,
thread: ThreadLock,
#[cfg(debug_assertions)]
debug: DebugData<Cell<u32>>,
_phantom: PhantomData<*const T>,
}
const DEBUG_STACK_TRACE: bool = false;
impl<T: RefCounted> RefCount<T> {
pub fn init() -> Self {
Self::init_exact_refs(1)
}
pub fn init_exact_refs(count: u32) -> Self {
debug_assert!(count > 0);
Self {
raw_count: Cell::new(count),
thread: ThreadLock::init_locked_if_non_comptime(),
#[cfg(debug_assertions)]
debug: DebugData::empty(),
_phantom: PhantomData,
}
}
pub unsafe fn ref_(self_: *mut T) {
let count = unsafe { &*T::get_ref_count(self_) };
#[cfg(debug_assertions)]
{
count.debug.assert_valid();
}
bun_core::scoped_log!(
ref_count,
"0x{:x} ref {} -> {}:",
self_ as usize,
count.raw_count.get(),
count.raw_count.get() + 1,
);
if DEBUG_STACK_TRACE {
dump_stack_hook(None, return_address());
}
count.assert_single_threaded();
count.raw_count.set(count.raw_count.get() + 1);
}
pub unsafe fn deref(self_: *mut T)
where
T: RefCounted<DestructorCtx = ()>,
{
unsafe { Self::deref_with_context(self_, ()) }
}
pub unsafe fn deref_with_context(self_: *mut T, ctx: T::DestructorCtx) {
let count = unsafe { &*T::get_ref_count(self_) };
#[cfg(debug_assertions)]
{
count.debug.assert_valid(); }
bun_core::scoped_log!(
ref_count,
"0x{:x} deref {} -> {}:",
self_ as usize,
count.raw_count.get(),
count.raw_count.get() - 1,
);
if DEBUG_STACK_TRACE {
dump_stack_hook(None, return_address());
}
count.assert_single_threaded();
count.raw_count.set(count.raw_count.get() - 1);
if count.raw_count.get() == 0 {
#[cfg(debug_assertions)]
{
unsafe { (*T::get_ref_count(self_)).debug.deinit(return_address()) };
}
unsafe { T::destructor(self_, ctx) };
}
}
pub unsafe fn dupe_ref(self_: *mut T) -> RefPtr<T>
where
T: AnyRefCounted,
{
unsafe { RefPtr::init_ref(self_) }
}
pub fn has_one_ref(&self) -> bool {
self.assert_single_threaded();
self.raw_count.get() == 1
}
pub fn get(&self) -> u32 {
self.raw_count.get()
}
pub fn dump_active_refs(&mut self) {
#[cfg(debug_assertions)]
{
let ptr: *mut T = unsafe {
bun_core::container_of::<T, Self>(std::ptr::from_mut(self), offset_of_ref_count())
};
self.debug.dump(
Some(core::any::type_name::<T>().as_bytes()),
ptr.cast::<c_void>(),
self.raw_count.get(),
);
}
}
pub fn assert_no_refs(&self) {
assert!(self.raw_count.get() == 0);
}
pub fn clear_without_destructor(&self) {
self.assert_single_threaded();
self.raw_count.set(0);
}
fn assert_single_threaded(&self) {
self.thread.lock_or_assert();
}
}
impl<T: RefCounted> AnyRefCounted for T {
type DestructorCtx = <T as RefCounted>::DestructorCtx;
unsafe fn rc_ref(this: *mut Self) {
unsafe { RefCount::<T>::ref_(this) }
}
unsafe fn rc_deref_with_context(this: *mut Self, ctx: Self::DestructorCtx) {
unsafe { RefCount::<T>::deref_with_context(this, ctx) }
}
unsafe fn rc_has_one_ref(this: *const Self) -> bool {
unsafe { (*T::get_ref_count(this.cast_mut())).has_one_ref() }
}
unsafe fn rc_assert_no_refs(this: *const Self) {
unsafe { (*T::get_ref_count(this.cast_mut())).assert_no_refs() }
}
#[cfg(debug_assertions)]
unsafe fn rc_debug_data(this: *mut Self) -> *mut dyn DebugDataOps {
unsafe { &raw mut (*T::get_ref_count(this)).debug }
}
}
pub struct ThreadSafeRefCount<T: ThreadSafeRefCounted> {
raw_count: AtomicU32,
#[cfg(debug_assertions)]
debug: DebugData<AtomicU32>,
_phantom: PhantomData<*const T>,
}
impl<T: ThreadSafeRefCounted> ThreadSafeRefCount<T> {
pub fn init() -> Self {
Self::init_exact_refs(1)
}
pub fn init_exact_refs(count: u32) -> Self {
debug_assert!(count > 0);
Self {
raw_count: AtomicU32::new(count),
#[cfg(debug_assertions)]
debug: DebugData::empty(),
_phantom: PhantomData,
}
}
pub unsafe fn ref_(self_: *mut T) {
let count = unsafe { &*T::get_ref_count(self_) };
#[cfg(debug_assertions)]
count.debug.assert_valid();
let old_count = count.raw_count.fetch_add(1, Ordering::SeqCst);
bun_core::scoped_log!(
ref_count,
"0x{:x} ref {} -> {}",
self_ as usize,
old_count,
old_count + 1,
);
debug_assert!(old_count > 0);
}
pub unsafe fn deref(self_: *mut T) {
let count = unsafe { &*T::get_ref_count(self_) };
#[cfg(debug_assertions)]
count.debug.assert_valid();
let old_count = count.raw_count.fetch_sub(1, Ordering::SeqCst);
bun_core::scoped_log!(
ref_count,
"0x{:x} deref {} -> {}",
self_ as usize,
old_count,
old_count - 1,
);
debug_assert!(old_count > 0);
if old_count == 1 {
#[cfg(debug_assertions)]
{
unsafe { (*T::get_ref_count(self_)).debug.deinit(return_address()) };
}
unsafe { T::destructor(self_) };
}
}
pub unsafe fn release(self_: *mut T) -> bool {
let count = unsafe { &*T::get_ref_count(self_) };
#[cfg(debug_assertions)]
count.debug.assert_valid();
let old_count = count.raw_count.fetch_sub(1, Ordering::SeqCst);
bun_core::scoped_log!(
ref_count,
"0x{:x} deref {} -> {}",
self_ as usize,
old_count,
old_count - 1,
);
debug_assert!(old_count > 0);
if old_count == 1 {
#[cfg(debug_assertions)]
{
unsafe { (*T::get_ref_count(self_)).debug.deinit(return_address()) };
}
true
} else {
false
}
}
pub unsafe fn dupe_ref(self_: *mut T) -> RefPtr<T>
where
T: AnyRefCounted,
{
#[cfg(debug_assertions)]
unsafe {
(*T::get_ref_count(self_)).debug.assert_valid();
}
unsafe { RefPtr::init_ref(self_) }
}
pub fn get(&self) -> u32 {
self.raw_count.load(Ordering::SeqCst)
}
pub fn has_one_ref(&self) -> bool {
#[cfg(debug_assertions)]
self.debug.assert_valid();
self.get() == 1
}
pub fn dump_active_refs(&mut self) {
#[cfg(debug_assertions)]
{
let ptr: *mut T = unsafe {
bun_core::container_of::<T, Self>(
std::ptr::from_mut(self),
offset_of_ref_count_ts(),
)
};
self.debug.dump(
Some(core::any::type_name::<T>().as_bytes()),
ptr.cast::<c_void>(),
self.raw_count.load(Ordering::SeqCst),
);
}
}
pub fn assert_no_refs(&self) {
assert!(self.raw_count.load(Ordering::SeqCst) == 0);
}
pub fn clear_without_destructor(&self) {
self.raw_count.store(0, Ordering::Relaxed);
}
#[cfg(debug_assertions)]
#[doc(hidden)]
#[inline]
pub fn debug_data_ptr(&mut self) -> *mut dyn DebugDataOps {
&raw mut self.debug
}
}
pub unsafe trait CellRefCounted: Sized {
fn ref_count(&self) -> &Cell<u32>;
unsafe fn ref_count_raw<'a>(this: *const Self) -> &'a Cell<u32>;
#[inline]
unsafe fn destroy(this: *mut Self) {
drop(unsafe { Box::from_raw(this) });
}
#[inline]
fn ref_(&self) {
let rc = self.ref_count();
rc.set(rc.get() + 1);
}
#[inline]
unsafe fn deref(this: *mut Self) {
let rc = unsafe { Self::ref_count_raw(this) };
let n = rc.get() - 1;
rc.set(n);
if n == 0 {
unsafe { Self::destroy(this) };
}
}
}
#[cfg(debug_assertions)]
#[doc(hidden)]
pub struct NoopDebugData;
#[cfg(debug_assertions)]
impl DebugDataOps for NoopDebugData {
fn assert_valid_dyn(&self) {}
fn acquire(&mut self, _return_address: usize) -> TrackedRefId {
TrackedRefId::new(0)
}
fn release(&mut self, _id: TrackedRefId, _return_address: usize) {}
}
#[cfg(debug_assertions)]
#[doc(hidden)]
pub fn noop_debug_data() -> *mut dyn DebugDataOps {
thread_local! {
static NOOP: core::cell::UnsafeCell<NoopDebugData> =
const { core::cell::UnsafeCell::new(NoopDebugData) };
}
NOOP.with(|n| n.get() as *mut dyn DebugDataOps)
}
pub struct RefPtr<T: AnyRefCounted> {
pub data: NonNull<T>,
#[cfg(debug_assertions)]
debug: TrackedRefId,
}
impl<T: AnyRefCounted> RefPtr<T> {
pub unsafe fn init_ref(raw_ptr: *mut T) -> Self {
unsafe { T::rc_ref(raw_ptr) };
unsafe { Self::unchecked_and_unsafe_init(raw_ptr, return_address()) }
}
pub fn deref(&self)
where
T::DestructorCtx: Default,
{
self.deref_with_context(Default::default());
}
pub fn deref_with_context(&self, ctx: T::DestructorCtx) {
#[cfg(debug_assertions)]
{
unsafe { (*T::rc_debug_data(self.as_ptr())).release(self.debug, return_address()) };
}
unsafe { T::rc_deref_with_context(self.as_ptr(), ctx) };
}
pub fn dupe_ref(&self) -> Self {
unsafe { Self::init_ref(self.as_ptr()) }
}
pub fn new(init_data: T) -> Self {
unsafe { Self::adopt_ref(bun_core::heap::into_raw(Box::new(init_data))) }
}
pub unsafe fn adopt_ref(raw_ptr: *mut T) -> Self {
#[cfg(debug_assertions)]
{
debug_assert!(unsafe { T::rc_has_one_ref(raw_ptr) });
unsafe { (*T::rc_debug_data(raw_ptr)).assert_valid_dyn() };
}
unsafe { Self::unchecked_and_unsafe_init(raw_ptr, return_address()) }
}
#[inline]
pub unsafe fn from_raw(raw_ptr: *mut T) -> Self {
unsafe { Self::take_ref(raw_ptr) }
}
#[inline]
pub fn into_raw(self) -> *mut T {
self.leak()
}
#[inline]
pub fn as_ptr(&self) -> *mut T {
self.data.as_ptr()
}
#[inline]
pub fn data(&self) -> &T {
unsafe { self.data.as_ref() }
}
pub unsafe fn take_ref(raw_ptr: *mut T) -> Self {
#[cfg(debug_assertions)]
{
unsafe { (*T::rc_debug_data(raw_ptr)).assert_valid_dyn() };
}
unsafe { Self::unchecked_and_unsafe_init(raw_ptr, return_address()) }
}
pub fn leak(self) -> *mut T {
let ptr = self.data.as_ptr();
#[cfg(debug_assertions)]
{
unsafe { (*T::rc_debug_data(ptr)).release(self.debug, return_address()) };
}
ptr
}
pub unsafe fn unchecked_and_unsafe_init(raw_ptr: *mut T, ret_addr: usize) -> Self {
let _ = ret_addr;
Self {
data: unsafe { NonNull::new_unchecked(raw_ptr) },
#[cfg(debug_assertions)]
debug: unsafe { (*T::rc_debug_data(raw_ptr)).acquire(ret_addr) },
}
}
}
impl<T: AnyRefCounted> Clone for RefPtr<T> {
#[inline]
fn clone(&self) -> Self {
self.dupe_ref()
}
}
impl<T: AnyRefCounted> core::ops::Deref for RefPtr<T> {
type Target = T;
#[inline]
fn deref(&self) -> &T {
self.data()
}
}
#[must_use = "dropping immediately releases the ref"]
pub struct ScopedRef<T: AnyRefCounted>(NonNull<T>)
where
T::DestructorCtx: Default;
impl<T: AnyRefCounted> ScopedRef<T>
where
T::DestructorCtx: Default,
{
#[inline]
pub unsafe fn new(ptr: *mut T) -> Self {
unsafe { T::rc_ref(ptr) };
Self(unsafe { NonNull::new_unchecked(ptr) })
}
#[inline]
pub unsafe fn adopt(ptr: *mut T) -> Self {
Self(unsafe { NonNull::new_unchecked(ptr) })
}
}
impl<T: AnyRefCounted> Drop for ScopedRef<T>
where
T::DestructorCtx: Default,
{
#[inline]
fn drop(&mut self) {
unsafe { T::rc_deref(self.0.as_ptr()) };
}
}
#[cfg(debug_assertions)]
struct TrackedRef {
acquired_at: StoredTrace,
}
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
pub struct TrackedRefId(u32);
#[cfg(debug_assertions)]
impl TrackedRefId {
#[inline]
const fn new(n: u32) -> Self {
Self(n)
}
}
#[cfg(debug_assertions)]
struct TrackedDeref;
#[cfg(debug_assertions)]
pub trait DebugDataOps {
fn assert_valid_dyn(&self);
fn acquire(&mut self, return_address: usize) -> TrackedRefId;
fn release(&mut self, id: TrackedRefId, return_address: usize);
}
#[cfg(debug_assertions)]
const MAGIC_VALID: u128 = 0x2f84_e51d;
#[cfg(debug_assertions)]
pub struct DebugData<Count> {
magic: u128,
lock: bun_core::Mutex<()>,
next_id: AtomicU32,
map: HashMap<TrackedRefId, TrackedRef>,
frees: ArrayHashMap<TrackedRefId, TrackedDeref>,
_count: core::marker::PhantomData<Count>,
}
#[cfg(debug_assertions)]
impl<Count: CountLoad> DebugData<Count> {
pub fn empty() -> Self {
Self {
magic: MAGIC_VALID,
lock: bun_core::Mutex::new(()),
next_id: AtomicU32::new(0),
map: HashMap::new(),
frees: ArrayHashMap::new(),
_count: core::marker::PhantomData,
}
}
fn assert_valid(&self) {
debug_assert!(self.magic == MAGIC_VALID);
}
fn dump(&mut self, type_name: Option<&[u8]>, ptr: *mut c_void, rc: u32) {
let _guard = self.lock.lock();
generic_dump(type_name, ptr, rc as usize, &mut self.map);
}
fn alloc_id(&self) -> TrackedRefId {
TrackedRefId::new(self.next_id.fetch_add(1, Ordering::SeqCst))
}
fn release_impl(&mut self, id: TrackedRefId, return_address: usize) {
let _guard = self.lock.lock();
let _ = return_address;
if self.map.remove(&id).is_none() {
return;
}
self.frees.insert(id, TrackedDeref);
}
fn deinit(&mut self, ret_addr: usize) {
self.assert_valid();
self.magic = 0; let _guard = self.lock.lock();
self.map.clear();
self.map.shrink_to_fit();
self.frees.clear();
let _ = ret_addr;
}
}
#[cfg(debug_assertions)]
impl<Count: CountLoad> DebugDataOps for DebugData<Count> {
fn assert_valid_dyn(&self) {
self.assert_valid();
}
fn acquire(&mut self, return_address: usize) -> TrackedRefId {
let _guard = self.lock.lock();
let id = self.alloc_id();
self.map.insert(
id,
TrackedRef {
acquired_at: StoredTrace::capture(Some(return_address)),
},
);
id
}
fn release(&mut self, id: TrackedRefId, return_address: usize) {
self.release_impl(id, return_address);
}
}
#[cfg(debug_assertions)]
pub trait CountLoad {
fn load_count(&self) -> u32;
}
#[cfg(debug_assertions)]
impl CountLoad for Cell<u32> {
fn load_count(&self) -> u32 {
self.get()
}
}
#[cfg(debug_assertions)]
impl CountLoad for AtomicU32 {
fn load_count(&self) -> u32 {
self.load(Ordering::SeqCst)
}
}
#[cfg(debug_assertions)]
fn generic_dump(
type_name: Option<&[u8]>,
ptr: *mut c_void,
total_ref_count: usize,
map: &mut HashMap<TrackedRefId, TrackedRef>,
) {
let tracked_refs = map.len();
let untracked_refs = total_ref_count - tracked_refs;
bun_core::pretty_error!(
"<blue>{}{}{:x} has ",
bstr::BStr::new(type_name.unwrap_or(b"")),
if type_name.is_some() { "@" } else { "" },
ptr as usize,
);
if tracked_refs > 0 {
bun_core::pretty_error!(
"{} tracked{}",
tracked_refs,
if untracked_refs > 0 { ", " } else { "" },
);
}
if untracked_refs > 0 {
bun_core::pretty_error!("{} untracked refs<r>\n", untracked_refs);
} else {
bun_core::pretty_error!("refs<r>\n");
}
let mut i: usize = 0;
for entry in map.values() {
bun_core::pretty_error!("<b>RefPtr acquired at:<r>\n");
dump_stack_hook(Some(&entry.acquired_at), 0);
i += 1;
if i >= 3 {
bun_core::pretty_error!(" {} omitted ...\n", map.len() - i);
break;
}
}
}
pub fn maybe_assert_no_refs<T: AnyRefCounted>(ptr: &T) {
unsafe { T::rc_assert_no_refs(std::ptr::from_ref::<T>(ptr)) }
}
#[inline(always)]
fn return_address() -> usize {
bun_core::return_address()
}
#[cfg(debug_assertions)]
#[inline(always)]
fn offset_of_ref_count() -> usize {
0
}
#[cfg(debug_assertions)]
#[inline(always)]
fn offset_of_ref_count_ts() -> usize {
0
}
bun_core::declare_scope!(ref_count, hidden);