use indexmap::IndexMap;
use std::cell::Cell;
use std::fmt;
use std::sync::Arc;
use slotmap::{SlotMap, new_key_type};
use super::Bytecode;
use super::BytecodeRuntime;
use super::LuaType;
use super::Table;
use super::Val;
#[derive(Clone, Debug)]
pub(crate) enum Upvalue {
Open(usize),
Closed(Val),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct UpvalueRef(u32);
impl UpvalueRef {
pub(crate) fn new(idx: u32) -> Self {
Self(idx)
}
pub(crate) fn index(self) -> usize {
self.0 as usize
}
}
pub(crate) struct UpvaluePool {
slots: Vec<Upvalue>,
}
impl Default for UpvaluePool {
fn default() -> Self {
Self::new()
}
}
impl UpvaluePool {
pub(super) fn new() -> Self {
Self {
slots: Vec::with_capacity(64),
}
}
#[hotpath::measure]
pub(super) fn alloc(&mut self, upvalue: Upvalue) -> UpvalueRef {
let idx = self.slots.len() as u32;
self.slots.push(upvalue);
UpvalueRef::new(idx)
}
#[cfg(feature = "snapshot")]
pub(super) fn alloc_closed_nil(&mut self) -> UpvalueRef {
self.alloc(Upvalue::Closed(Val::Nil))
}
#[cfg(feature = "snapshot")]
pub(super) fn set_closed(&mut self, uv_ref: UpvalueRef, val: Val) {
self.slots[uv_ref.index()] = Upvalue::Closed(val);
}
#[inline]
pub(super) fn get(&self, uv_ref: UpvalueRef) -> &Upvalue {
&self.slots[uv_ref.index()]
}
#[inline]
pub(super) fn get_mut(&mut self, uv_ref: UpvalueRef) -> &mut Upvalue {
&mut self.slots[uv_ref.index()]
}
}
#[derive(Clone, Debug)]
pub(super) struct Closure {
pub(super) bytecode: Arc<Bytecode>,
pub(super) runtime: Arc<BytecodeRuntime>,
pub(super) upvalues: Arc<[UpvalueRef]>,
}
pub(super) enum RawObject {
LuaFn(Box<Closure>),
Table(Table),
}
impl RawObject {
#[must_use]
pub(super) const fn typ(&self) -> LuaType {
match self {
RawObject::LuaFn(_) => LuaType::Function,
RawObject::Table(_) => LuaType::Table,
}
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Color {
Unmarked,
Reachable,
}
pub(super) struct WrappedObject {
pub(super) raw: RawObject,
pub(super) color: Cell<Color>,
}
new_key_type! {
pub struct ObjectKey;
}
new_key_type! {
pub struct StringKey;
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub(crate) struct ObjectPtr(pub(crate) ObjectKey);
impl ObjectPtr {
pub(super) fn typ(self, heap: &GcHeap) -> LuaType {
heap.get(self).raw.typ()
}
}
pub(crate) struct GcHeap {
objects: SlotMap<ObjectKey, WrappedObject>,
threshold: usize,
strings: StringPool,
mark_worklist: Vec<ObjectPtr>,
empty_upvalues: Arc<[UpvalueRef]>,
}
impl GcHeap {
pub(super) fn with_threshold(threshold: usize) -> Self {
Self {
objects: SlotMap::with_key(),
threshold,
strings: StringPool::new(),
mark_worklist: Vec::new(),
empty_upvalues: Arc::from([]),
}
}
pub(super) fn reserve(&mut self, additional_objects: usize, additional_strings: usize) {
self.objects.reserve(additional_objects);
self.strings.reserve(additional_strings);
}
#[inline]
pub(super) fn get(&self, ptr: ObjectPtr) -> &WrappedObject {
self.objects
.get(ptr.0)
.expect("Invalid ObjectPtr: object was freed (use-after-free detected)")
}
#[inline]
pub(super) fn get_mut(&mut self, ptr: ObjectPtr) -> &mut WrappedObject {
self.objects
.get_mut(ptr.0)
.expect("Invalid ObjectPtr: object was freed (use-after-free detected)")
}
#[hotpath::measure]
pub(super) fn as_lua_function(&self, ptr: ObjectPtr) -> Option<Closure> {
match &self.get(ptr).raw {
RawObject::LuaFn(closure) => Some((**closure).clone()),
_ => None,
}
}
pub(super) fn as_table(&mut self, ptr: ObjectPtr) -> Option<&mut Table> {
match &mut self.get_mut(ptr).raw {
RawObject::Table(t) => Some(t),
_ => None,
}
}
pub(super) fn as_table_ref(&self, ptr: ObjectPtr) -> Option<&Table> {
match &self.get(ptr).raw {
RawObject::Table(t) => Some(t),
_ => None,
}
}
pub(super) fn get_string(&self, ptr: StringPtr) -> &[u8] {
self.strings.get(ptr)
}
#[hotpath::measure]
pub(super) fn alloc_lua_fn(
&mut self,
bytecode: Arc<Bytecode>,
runtime: Arc<BytecodeRuntime>,
upvalues: Vec<UpvalueRef>,
) -> ObjectPtr {
let upvalues = if upvalues.is_empty() {
Arc::clone(&self.empty_upvalues)
} else {
upvalues.into()
};
let closure = Closure {
bytecode,
runtime,
upvalues,
};
let raw = RawObject::LuaFn(Box::new(closure));
let wrapped = WrappedObject {
raw,
color: Cell::new(Color::Unmarked),
};
ObjectPtr(self.objects.insert(wrapped))
}
#[hotpath::measure]
pub(super) fn alloc_table(&mut self) -> ObjectPtr {
let raw = RawObject::Table(Table::default());
let wrapped = WrappedObject {
raw,
color: Cell::new(Color::Unmarked),
};
ObjectPtr(self.objects.insert(wrapped))
}
pub(super) fn alloc_table_with_capacity(&mut self, capacity: usize) -> ObjectPtr {
let raw = RawObject::Table(Table::with_capacity(capacity));
let wrapped = WrappedObject {
raw,
color: Cell::new(Color::Unmarked),
};
ObjectPtr(self.objects.insert(wrapped))
}
pub(super) fn alloc_table_with_template(
&mut self,
key_ids: &[u16],
literals: &[Val],
) -> ObjectPtr {
let raw = RawObject::Table(Table::with_template_keys(key_ids, literals));
let wrapped = WrappedObject {
raw,
color: Cell::new(Color::Unmarked),
};
ObjectPtr(self.objects.insert(wrapped))
}
#[hotpath::measure]
pub(super) fn alloc_string(&mut self, bytes: &[u8]) -> StringPtr {
let hash = StringPool::hash_string(bytes);
if let Some(ptr) = self.strings.find_by_hash(bytes, hash) {
return ptr;
}
self.strings.insert_with_hash(bytes.into(), hash)
}
#[must_use]
pub(super) fn is_full(&self) -> bool {
self.allocation_count() >= self.threshold
}
#[hotpath::measure]
pub(super) fn mark(&self, ptr: ObjectPtr, worklist: &mut Vec<ObjectPtr>) {
if let Some(obj) = self.objects.get(ptr.0)
&& obj.color.get() == Color::Unmarked
{
obj.color.set(Color::Reachable);
worklist.push(ptr);
}
}
pub(super) fn take_mark_worklist(&mut self) -> Vec<ObjectPtr> {
let mut worklist = std::mem::take(&mut self.mark_worklist);
worklist.clear();
worklist
}
pub(super) fn restore_mark_worklist(&mut self, mut worklist: Vec<ObjectPtr>) {
worklist.clear();
self.mark_worklist = worklist;
}
pub(super) fn drain_mark_worklist(
&self,
worklist: &mut Vec<ObjectPtr>,
upvalue_pool: &UpvaluePool,
) {
while let Some(ptr) = worklist.pop() {
debug_assert!(
self.objects
.get(ptr.0)
.is_none_or(|obj| obj.color.get() == Color::Reachable),
"GC worklist entry was not marked reachable; it was pushed \
directly instead of through GcHeap::mark"
);
self.mark_children(self.get(ptr), upvalue_pool, worklist);
}
}
pub(super) fn mark_string(&self, ptr: StringPtr) {
self.strings.mark(ptr);
}
#[hotpath::measure]
fn mark_children(
&self,
obj: &WrappedObject,
upvalue_pool: &UpvaluePool,
worklist: &mut Vec<ObjectPtr>,
) {
match &obj.raw {
RawObject::LuaFn(closure) => {
closure.runtime.literals.mark_reachable(self, worklist);
for uv_ref in closure.upvalues.iter() {
if let Upvalue::Closed(val) = upvalue_pool.get(*uv_ref) {
val.mark_reachable(self, worklist);
}
}
}
RawObject::Table(tbl) => {
tbl.mark_values(self, worklist);
}
}
}
#[hotpath::measure(label = "object::heap_collect")]
pub(super) fn collect(&mut self) {
#[cfg(feature = "debug_gc")]
{
println!("Running garbage collector");
println!("Initial size: {}", self.objects.len());
}
self.objects.retain(|_, obj| match obj.color.get() {
Color::Reachable => {
obj.color.set(Color::Unmarked);
true
}
Color::Unmarked => false,
});
self.strings.collect();
if self.threshold != usize::MAX {
self.threshold = self.allocation_count().saturating_mul(2).max(20);
}
#[cfg(feature = "debug_gc")]
println!("Final size: {}", self.objects.len());
}
pub(super) fn reachable_lua_bytecodes(&self) -> Vec<Arc<Bytecode>> {
self.objects
.values()
.filter(|object| object.color.get() == Color::Reachable)
.filter_map(|object| match &object.raw {
RawObject::LuaFn(closure) => Some(Arc::clone(&closure.bytecode)),
RawObject::Table(_) => None,
})
.collect()
}
pub(super) fn object_count(&self) -> usize {
self.objects.len()
}
pub(super) fn string_count(&self) -> usize {
self.strings.len()
}
#[inline]
pub(super) fn allocation_count(&self) -> usize {
self.objects.len().saturating_add(self.strings.len())
}
pub(super) fn threshold(&self) -> usize {
self.threshold
}
pub(super) fn set_threshold(&mut self, threshold: usize) {
self.threshold = threshold;
}
}
pub(super) trait Markable {
fn mark_reachable(&self, heap: &GcHeap, worklist: &mut Vec<ObjectPtr>);
}
impl Markable for Val {
fn mark_reachable(&self, heap: &GcHeap, worklist: &mut Vec<ObjectPtr>) {
match self {
Val::Obj(ptr) => heap.mark(*ptr, worklist),
Val::Str(ptr) => heap.mark_string(*ptr),
_ => (),
}
}
}
impl<T: Markable> Markable for [T] {
fn mark_reachable(&self, heap: &GcHeap, worklist: &mut Vec<ObjectPtr>) {
for val in self {
val.mark_reachable(heap, worklist);
}
}
}
impl<K, V: Markable> Markable for IndexMap<K, V> {
fn mark_reachable(&self, heap: &GcHeap, worklist: &mut Vec<ObjectPtr>) {
for val in self.values() {
val.mark_reachable(heap, worklist);
}
}
}
struct StringEntry {
data: Box<[u8]>,
hash: u64,
color: Cell<Color>,
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub(crate) struct StringPtr(StringKey);
impl fmt::Display for StringPtr {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "string: {:?}", self.0)
}
}
pub(crate) struct StringPool {
strings: SlotMap<StringKey, StringEntry>,
hash_index: IndexMap<u64, Vec<StringKey>>,
}
impl StringPool {
fn new() -> Self {
Self {
strings: SlotMap::with_key(),
hash_index: IndexMap::new(),
}
}
fn reserve(&mut self, additional: usize) {
self.strings.reserve(additional);
self.hash_index.reserve(additional);
}
pub(super) fn len(&self) -> usize {
self.strings.len()
}
pub(super) fn hash_string(bytes: &[u8]) -> u64 {
const FX_HASH_MUL: u64 = 0x517cc1b727220a95;
#[inline]
fn mix(hash: u64, word: u64) -> u64 {
(hash.rotate_left(5) ^ word).wrapping_mul(FX_HASH_MUL)
}
let mut hash = bytes.len() as u64;
let (chunks, remainder) = bytes.as_chunks::<8>();
for chunk in chunks {
let word = u64::from_le_bytes(*chunk);
hash = mix(hash, word);
}
let mut tail = 0u64;
for (i, byte) in remainder.iter().enumerate() {
tail |= u64::from(*byte) << (i * 8);
}
if !remainder.is_empty() {
hash = mix(hash, tail);
}
hash
}
pub(super) fn get(&self, ptr: StringPtr) -> &[u8] {
&self
.strings
.get(ptr.0)
.expect("Invalid StringPtr: string was freed (use-after-free detected)")
.data
}
#[hotpath::measure]
pub(super) fn find_by_hash(&self, bytes: &[u8], hash: u64) -> Option<StringPtr> {
let bucket = self.hash_index.get(&hash)?;
for key in bucket {
if let Some(entry) = self.strings.get(*key)
&& entry.data.as_ref() == bytes
{
return Some(StringPtr(*key));
}
}
None
}
#[hotpath::measure]
pub(super) fn insert_with_hash(&mut self, bytes: Box<[u8]>, hash: u64) -> StringPtr {
let entry = StringEntry {
data: bytes,
hash,
color: Cell::new(Color::Unmarked),
};
let key = self.strings.insert(entry);
self.hash_index.entry(hash).or_default().push(key);
StringPtr(key)
}
pub(super) fn mark(&self, ptr: StringPtr) {
if let Some(entry) = self.strings.get(ptr.0) {
entry.color.set(Color::Reachable);
}
}
#[hotpath::measure(label = "object::string_pool_collect")]
pub(super) fn collect(&mut self) {
let mut removed: Vec<(StringKey, u64)> = Vec::new();
self.strings.retain(|key, entry| match entry.color.get() {
Color::Reachable => {
entry.color.set(Color::Unmarked);
true
}
Color::Unmarked => {
removed.push((key, entry.hash));
false
}
});
for (key, hash) in removed {
if let Some(bucket) = self.hash_index.get_mut(&hash) {
bucket.retain(|k| *k != key);
if bucket.is_empty() {
self.hash_index.shift_remove(&hash);
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_allocation() {
let mut heap = GcHeap::with_threshold(100);
let t1 = heap.alloc_table();
let t2 = heap.alloc_table();
assert!(heap.as_table_ref(t1).is_some());
assert!(heap.as_table_ref(t2).is_some());
assert_eq!(heap.object_count(), 2);
}
#[test]
fn test_gc_collect() {
let mut heap = GcHeap::with_threshold(100);
let kept = heap.alloc_table();
let _freed = heap.alloc_table();
let mut worklist = heap.take_mark_worklist();
heap.mark(kept, &mut worklist);
heap.drain_mark_worklist(&mut worklist, &UpvaluePool::new());
heap.restore_mark_worklist(worklist);
heap.collect();
assert!(heap.as_table_ref(kept).is_some());
assert_eq!(heap.object_count(), 1);
}
#[test]
fn deep_table_chain_marks_iteratively_and_reuses_the_empty_worklist() {
let mut heap = GcHeap::with_threshold(100);
let mut root = heap.alloc_table();
for _ in 0..100_000 {
let next = heap.alloc_table();
heap.as_table(next)
.expect("newly allocated object is a table")
.insert(Val::Num(1.0), Val::Obj(root))
.expect("table key is valid");
root = next;
}
for _ in 0..2 {
let mut worklist = heap.take_mark_worklist();
heap.mark(root, &mut worklist);
heap.drain_mark_worklist(&mut worklist, &UpvaluePool::new());
assert!(worklist.is_empty());
heap.restore_mark_worklist(worklist);
heap.collect();
assert_eq!(heap.object_count(), 100_001);
assert!(heap.mark_worklist.is_empty());
}
}
#[test]
#[should_panic(expected = "use-after-free")]
fn test_use_after_free_detection() {
let mut heap = GcHeap::with_threshold(100);
let ptr = heap.alloc_table();
heap.collect();
let _ = heap.as_table_ref(ptr);
}
#[test]
fn test_string_allocation() {
let mut heap = GcHeap::with_threshold(100);
let s1 = heap.alloc_string(b"hello");
let s2 = heap.alloc_string(b"world");
let s3 = heap.alloc_string(b"hello");
assert_eq!(heap.get_string(s1), b"hello");
assert_eq!(heap.get_string(s2), b"world");
assert_eq!(s1, s3); assert_eq!(heap.string_count(), 2);
}
#[test]
fn test_string_hash_is_pinned() {
assert_eq!(StringPool::hash_string(b""), 0x0000000000000000);
assert_eq!(StringPool::hash_string(b"hello"), 0xd76e0ef553a10d68);
}
#[test]
fn test_string_gc_collect() {
let mut heap = GcHeap::with_threshold(100);
let kept = heap.alloc_string(b"keep");
let _freed = heap.alloc_string(b"free");
heap.mark_string(kept);
heap.collect();
assert_eq!(heap.get_string(kept), b"keep");
assert_eq!(heap.string_count(), 1);
}
#[test]
#[should_panic(expected = "use-after-free")]
fn test_string_use_after_free_detection() {
let mut heap = GcHeap::with_threshold(100);
let ptr = heap.alloc_string(b"test");
heap.collect();
let _ = heap.get_string(ptr);
}
}