use crate::VmErrorResult;
use crate::gc::GcObject;
use crate::gc::GcRuntime;
use crate::gc::debug::{DumpContext, HeapEnumContext};
use crate::gc::{GCS_ATOMIC, GCS_PAUSE, GCS_PROPAGATE, GCS_PROPAGATE_AGAIN, GCS_SWEEP};
use crate::gc::{GcCategoryNamer, GcHeapEdge, GcHeapNode};
use crate::handle::RawHandle;
use crate::memory::MemoryRuntime;
use crate::state::GlobalState;
use crate::state::{GcInterrupt, GcPhase};
use crate::thread::Thread;
use crate::types;
use crate::value::TValue;
use luau_common::{BString, flags};
const GC_SWEEP_PAGE_STEP_COST: usize = 16;
impl GlobalState {
unsafe fn heap_trigger_error_offset(&self) -> i64 {
let global_mut = unsafe { self.as_ptr().as_mut().unwrap_unchecked() };
let stats = &mut global_mut.gc_stats;
let error_kb = (stats
.atomic_start_total_size_bytes
.wrapping_sub(stats.heap_goal_size_bytes)
/ 1024) as i32;
let slot =
&mut stats.trigger_terms[stats.trigger_term_pos as usize % stats.trigger_terms.len()];
let previous = *slot;
*slot = error_kb;
stats.trigger_integral += error_kb - previous;
stats.trigger_term_pos += 1;
let ku = 0.9f64;
let tu = 2.5f64;
let kp = 0.45 * ku;
let ti = 0.8 * tu;
let ki = 0.54 * ku / ti;
let proportional = kp * error_kb as f64;
let integral = ki * stats.trigger_integral as f64;
((proportional + integral) * 1024.0) as i64
}
unsafe fn heap_trigger(&self, heap_goal: usize) -> usize {
unsafe {
let stats = &self.as_ptr().as_ref().unwrap_unchecked().gc_stats;
let allocation_duration = stats.atomic_start_timestamp - stats.end_timestamp;
if allocation_duration < 1e-3 {
return heap_goal;
}
let allocation_rate = stats
.atomic_start_total_size_bytes
.wrapping_sub(stats.end_total_size_bytes) as f64
/ allocation_duration;
let mark_duration = stats.atomic_start_timestamp - stats.start_timestamp;
let expected_growth = (mark_duration * allocation_rate) as i64;
let offset = self.heap_trigger_error_offset();
let heap_trigger = heap_goal as i64 - (expected_growth + offset);
let total_bytes = self.as_ptr().as_ref().unwrap_unchecked().total_bytes as i64;
if heap_trigger < total_bytes {
total_bytes as usize
} else if heap_trigger > heap_goal as i64 {
heap_goal
} else {
heap_trigger as usize
}
}
}
}
impl Thread {
unsafe fn gc_interrupt(&self, event: GcInterrupt) -> VmErrorResult {
let global = unsafe { self.global() };
let Some(interrupt) = global.take_gc_interrupt_callback() else {
return Ok(());
};
interrupt(self, event)
}
unsafe fn gc_step(&self, limit: usize) -> usize {
unsafe {
let global = self.global();
let mut cost = 0usize;
match global.gc_state() {
GCS_PAUSE => {
self.mark_root();
debug_assert_eq!(global.gc_state(), GCS_PROPAGATE);
}
GCS_PROPAGATE => {
while global.gray().is_some() && cost < limit {
cost += global.propagate_mark();
}
if global.gray().is_none() {
global.set_gray(global.gray_again());
global.set_gray_again(None);
global.set_gc_state(GCS_PROPAGATE_AGAIN);
}
}
GCS_PROPAGATE_AGAIN => {
while global.gray().is_some() && cost < limit {
cost += global.propagate_mark();
}
if global.gray().is_none() {
global.set_gc_state(GCS_ATOMIC);
}
}
GCS_ATOMIC => {
let global_mut = global.as_ptr().as_mut().unwrap_unchecked();
global_mut.gc_stats.atomic_start_timestamp = crate::perf::clock();
global_mut.gc_stats.atomic_start_total_size_bytes = global_mut.total_bytes;
cost = self.atomic();
debug_assert_eq!(global.gc_state(), GCS_SWEEP);
}
GCS_SWEEP => {
while let Some(page) = global.sweep_gco_page()
&& cost < limit
{
let next = page.next_page();
let steps = page.sweep_gco(self);
global.set_sweep_gco_page(next);
cost += steps as usize * GC_SWEEP_PAGE_STEP_COST;
}
if global.sweep_gco_page().is_none() {
let main_thread = global.main_thread();
debug_assert!(!global.is_dead((&main_thread).into()));
global.make_white((&main_thread).into());
self.shrink_buffers();
global.set_gc_state(GCS_PAUSE);
}
}
other => unreachable!("unexpected gc state {}", other),
}
cost
}
}
}
impl GcRuntime for Thread {
unsafe fn free_all(&self) {
unsafe {
let global = self.global();
debug_assert!(*self == global.main_thread());
self.visit_gco(self.as_ptr().cast(), super::sweep::delete_gco);
for index in 0..global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.string_table
.size
.max(0) as usize
{
debug_assert!(
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.string_table
.hash
.add(index)
.read()
.is_null()
);
}
debug_assert_eq!(
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.string_table
.n_use,
0
);
}
}
unsafe fn needs_gc(&self) -> bool {
let global = unsafe { self.global() };
unsafe {
global.as_ptr().as_ref().unwrap_unchecked().total_bytes
>= global.as_ptr().as_ref().unwrap_unchecked().gc_threshold
}
}
unsafe fn check_gc(&self) -> VmErrorResult {
if unsafe { self.needs_gc() } {
unsafe { self.step(true)? };
}
Ok(())
}
unsafe fn step(&self, assist: bool) -> VmErrorResult<usize> {
unsafe {
let global = self.global();
let step_size = global.as_ptr().as_ref().unwrap_unchecked().gc_step_size as usize;
let step_mul = global.as_ptr().as_ref().unwrap_unchecked().gc_step_mul as usize;
let mut limit = step_size * step_mul / 100;
debug_assert!(
global.as_ptr().as_ref().unwrap_unchecked().total_bytes
>= global.as_ptr().as_ref().unwrap_unchecked().gc_threshold
);
let debt = global.as_ptr().as_ref().unwrap_unchecked().total_bytes
- global.as_ptr().as_ref().unwrap_unchecked().gc_threshold;
if flags::LuauBackedgeHeapCheck.get() && assist {
limit = limit.max(debt * step_mul / 100);
}
self.gc_interrupt(GcInterrupt::BeforeStep)?;
let gc_state = global.gc_state();
if gc_state == GCS_PAUSE {
global
.as_ptr()
.as_mut()
.unwrap_unchecked()
.gc_stats
.start_timestamp = crate::perf::clock();
}
let last_gc_state = gc_state;
let work = self.gc_step(limit);
let actual_step_size =
work * 100 / global.as_ptr().as_ref().unwrap_unchecked().gc_step_mul as usize;
if global.gc_state() == GCS_PAUSE {
let total_bytes = global.as_ptr().as_ref().unwrap_unchecked().total_bytes;
let gc_goal = global.as_ptr().as_ref().unwrap_unchecked().gc_goal as usize;
let heap_goal = (total_bytes / 100) * gc_goal;
let heap_trigger = global.heap_trigger(heap_goal);
let end_timestamp = crate::perf::clock();
let global_mut = global.as_ptr().as_mut().unwrap_unchecked();
global_mut.gc_threshold = heap_trigger;
global_mut.gc_stats.heap_goal_size_bytes = heap_goal;
global_mut.gc_stats.end_timestamp = end_timestamp;
global_mut.gc_stats.end_total_size_bytes = global_mut.total_bytes;
} else {
let global_mut = global.as_ptr().as_mut().unwrap_unchecked();
global_mut.gc_threshold = global_mut.total_bytes + actual_step_size;
if global_mut.gc_threshold >= debt {
global_mut.gc_threshold -= debt;
}
}
self.gc_interrupt(GcInterrupt::AfterStep {
previous_phase: GcPhase::from_state(last_gc_state),
})?;
Ok(actual_step_size)
}
}
unsafe fn full_gc(&self) {
unsafe {
let global = self.global();
if global.keep_invariant() {
global.set_sweep_gco_page(global.all_gco_pages());
global.set_gray(None);
global.set_gray_again(None);
global.set_weak(None);
global.set_gc_state(GCS_SWEEP);
}
debug_assert!(matches!(global.gc_state(), GCS_PAUSE | GCS_SWEEP));
while global.gc_state() != GCS_PAUSE {
debug_assert_eq!(global.gc_state(), GCS_SWEEP);
self.gc_step(usize::MAX);
}
let sentinel = global.uv_head();
let mut upvalue = sentinel.open_data().next();
while upvalue != sentinel {
let current_upvalue = upvalue;
let next = current_upvalue.open_data().next();
current_upvalue
.as_ptr()
.as_mut()
.unwrap_unchecked()
.marked_open = 0;
upvalue = next;
}
self.mark_root();
while global.gc_state() != GCS_PAUSE {
self.gc_step(usize::MAX);
}
self.shrink_buffers_full();
let total_bytes = global.as_ptr().as_ref().unwrap_unchecked().total_bytes;
let gc_goal = global.as_ptr().as_ref().unwrap_unchecked().gc_goal as usize;
let gc_step_mul = global.as_ptr().as_ref().unwrap_unchecked().gc_step_mul as usize;
let heap_goal_size_bytes = (total_bytes / 100) * gc_goal;
let mut gc_threshold = total_bytes * (gc_goal * gc_step_mul / 100 - 100) / gc_step_mul;
if gc_threshold < total_bytes {
gc_threshold = total_bytes;
}
let global_mut = global.as_ptr().as_mut().unwrap_unchecked();
global_mut.gc_threshold = gc_threshold;
global_mut.gc_stats.heap_goal_size_bytes = heap_goal_size_bytes;
}
}
unsafe fn validate(&self) {
unsafe {
let global = self.global();
debug_assert!(!global.is_dead(self.into()));
global.validate_liveness(TValue::from_ref(
&global.as_ptr().as_ref().unwrap_unchecked().registry,
));
for tag in 0..types::LUA_T_COUNT {
if let Some(metatable) = global.metatable(tag) {
debug_assert!(!global.is_dead(metatable.into()));
}
}
for metatable in (&*global.userdata_type_registry_ptr()).recognized_metatables() {
debug_assert!(!global.is_dead(metatable.into()));
}
for tag in 0..crate::userdata::USERDATA_TAG_LIMIT {
if let Some(metatable) = global.userdata_metatable(tag) {
debug_assert!(!global.is_dead(metatable.into()));
}
}
for tag in 0..crate::userdata::USERDATA_INTERNAL_LIMIT {
let direct_access =
&global.as_ptr().as_ref().unwrap_unchecked().userdata_direct[tag];
global.validate_liveness(TValue::from_ref(&direct_access.index_tm));
global.validate_liveness(TValue::from_ref(&direct_access.new_index_tm));
global.validate_liveness(TValue::from_ref(&direct_access.name_call_tm));
if let Some(fields) = global.userdata_direct_field(tag) {
debug_assert!(!global.is_dead(fields.into()));
}
}
global.validate_gray_list(global.weak());
global.validate_gray_list(global.gray());
global.validate_gray_list(global.gray_again());
global.validate_object(GcObject::from(self));
self.visit_gco(self.as_ptr().cast(), super::debug::validate_gco_visitor);
let sentinel = global.uv_head();
let mut upvalue = sentinel.open_data().next();
while upvalue != sentinel {
let current_upvalue = upvalue;
let open = current_upvalue.open_data();
let object: GcObject = current_upvalue.into();
debug_assert_eq!(
current_upvalue.as_ptr().as_ref().unwrap_unchecked().tt,
types::LUA_TUPVALUE as u8
);
debug_assert!(current_upvalue.is_open());
debug_assert!(open.next().open_data().prev() == current_upvalue);
debug_assert!(open.prev().open_data().next() == current_upvalue);
debug_assert!(!object.is_black());
upvalue = open.next();
}
}
}
unsafe fn dump(&self, file: *mut (), category_name: Option<&mut dyn GcCategoryNamer>) {
unsafe {
let global = self.global();
let output = &mut *file.cast::<BString>();
let mut category_name = category_name;
output.clear();
output.extend_from_slice(b"{\"objects\":{\n");
super::debug::dump_gco(output, self, global.main_thread().into());
let mut context = DumpContext {
thread: self,
output,
};
self.visit_gco((&raw mut context).cast(), super::debug::dump_gco_visitor);
output.extend_from_slice(b"\"0\":{\"type\":\"userdata\",\"cat\":0,\"size\":0}\n");
output.extend_from_slice(b"},\"roots\":{\n");
output.extend_from_slice(b"\"mainthread\":");
super::debug::append_ref(output, global.main_thread().into());
output.extend_from_slice(b",\"registry\":");
super::debug::append_ref(
output,
TValue::from_ref(&global.as_ptr().as_ref().unwrap_unchecked().registry).gc_value(),
);
output.extend_from_slice(b"},\"stats\":{\n");
output.extend_from_slice(b"\"size\":");
super::debug::append_decimal(
output,
global.as_ptr().as_ref().unwrap_unchecked().total_bytes,
);
output.extend_from_slice(b",\n\"categories\":{\n");
for (index, bytes) in global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.memcat_bytes
.iter()
.copied()
.enumerate()
{
if bytes == 0 {
continue;
}
output.push(b'"');
super::debug::append_decimal(output, index);
output.extend_from_slice(b"\":{");
if let Some(category_name) = category_name.as_deref_mut() {
output.extend_from_slice(b"\"name\":\"");
category_name.category_name(self, index as u8, output);
output.extend_from_slice(b"\", ");
}
output.extend_from_slice(b"\"size\":");
super::debug::append_decimal(output, bytes);
output.extend_from_slice(b"},\n");
}
output.extend_from_slice(b"\"none\":{}\n}\n}}\n");
}
}
unsafe fn enum_heap(&self, context: *mut (), node: GcHeapNode, edge: GcHeapEdge) {
unsafe {
let global = self.global();
let mut heap = HeapEnumContext {
thread: self,
context,
node,
edge,
};
heap.enum_object(global.main_thread().into());
self.visit_gco((&raw mut heap).cast(), super::debug::enum_heap_gco_visitor);
}
}
unsafe fn allocation_rate(&self) -> i64 {
unsafe {
let global = self.global();
let duration_threshold = 1e-3;
let (bytes, duration) = match global.gc_state() {
x if x <= GCS_ATOMIC => (
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.total_bytes
.wrapping_sub(
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.gc_stats
.end_total_size_bytes,
),
crate::perf::clock()
- global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.gc_stats
.end_timestamp,
),
_ => (
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.gc_stats
.atomic_start_total_size_bytes
.wrapping_sub(
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.gc_stats
.end_total_size_bytes,
),
global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.gc_stats
.atomic_start_timestamp
- global
.as_ptr()
.as_ref()
.unwrap_unchecked()
.gc_stats
.end_timestamp,
),
};
if duration < duration_threshold {
-1
} else {
(bytes as f64 / duration) as i64
}
}
}
}