#![allow(non_upper_case_globals, dead_code)]
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::fmt;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use crate::codegen::MachineInstr;
pub const X86_GC_MAX_ROOTS: usize = 512;
pub const X86_GC_MAX_SAFEPOINTS: usize = 1024;
pub const X86_GC_OBJECT_ALIGNMENT: u32 = 8;
pub const X86_GC_CARD_SIZE: u32 = 512;
pub const X86_GC_REMEMBERED_LOG_SIZE: usize = 12;
pub const X86_GC_STACK_MAP_VERSION: u8 = 3;
pub const X86_GC_STACK_MAP_SECTION: &str = ".llvm_stackmaps";
pub const X86_GC_DEFAULT_NURSERY_SIZE: usize = 4 * 1024 * 1024;
pub const X86_GC_DEFAULT_SEMISPACE_SIZE: usize = 8 * 1024 * 1024;
pub const X86_GC_DEFAULT_TLAB_SIZE: usize = 64 * 1024;
pub const X86_GC_TLAB_REFILL_THRESHOLD: usize = 4096;
pub const X86_GC_LARGE_OBJECT_THRESHOLD: usize = 85 * 1024;
pub const X86_GC_ERLANG_REDUCTIONS: u64 = 4000;
pub const X86_GC_POLLING_PAGE_ADDR: u64 = 0x7FFF_FFFF_FFFF_F000;
pub const X86_GC_POLLING_PAGE_SIZE: u64 = 4096;
pub const X86_GC_MAX_FINALIZERS_PER_CYCLE: usize = 1024;
pub const X86_GC_MAX_WEAK_REFS_PER_CYCLE: usize = 4096;
pub const X86_GC_HEAP_GROWTH_FACTOR: f64 = 1.5;
pub const X86_GC_MIN_HEAP_OCCUPANCY: f64 = 0.75;
const DWARF_REG_RAX: u16 = 0;
const DWARF_REG_RDX: u16 = 1;
const DWARF_REG_RCX: u16 = 2;
const DWARF_REG_RBX: u16 = 3;
const DWARF_REG_RSI: u16 = 4;
const DWARF_REG_RDI: u16 = 5;
const DWARF_REG_RBP: u16 = 6;
const DWARF_REG_RSP: u16 = 7;
const DWARF_REG_R8: u16 = 8;
const DWARF_REG_R9: u16 = 9;
const DWARF_REG_R10: u16 = 10;
const DWARF_REG_R11: u16 = 11;
const DWARF_REG_R12: u16 = 12;
const DWARF_REG_R13: u16 = 13;
const DWARF_REG_R14: u16 = 14;
const DWARF_REG_R15: u16 = 15;
const DWARF_REG_RIP: u16 = 16;
const CALLEE_SAVED_GPRS: &[u16] = &[
DWARF_REG_RBX,
DWARF_REG_RBP,
DWARF_REG_R12,
DWARF_REG_R13,
DWARF_REG_R14,
DWARF_REG_R15,
];
const CALLER_SAVED_GPRS: &[u16] = &[
DWARF_REG_RAX,
DWARF_REG_RCX,
DWARF_REG_RDX,
DWARF_REG_RSI,
DWARF_REG_RDI,
DWARF_REG_R8,
DWARF_REG_R9,
DWARF_REG_R10,
DWARF_REG_R11,
];
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum X86GCStrategy {
ShadowStack,
Statepoint,
CoreCLR,
Erlang,
OCaml,
Custom(X86GCCustomStrategy),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct X86GCCustomStrategy {
pub name: String,
pub uses_statepoints: bool,
pub uses_shadow_stack: bool,
pub supports_generational: bool,
pub requires_write_barriers: bool,
pub requires_read_barriers: bool,
pub supports_concurrent_marking: bool,
pub suspension: X86GCSuspension,
pub safepoint_frequency: X86GCSafepointFrequency,
pub write_barrier_kind: X86GCWriteBarrierKind,
pub read_barrier_kind: X86GCReadBarrierKind,
pub params: Vec<(String, String)>,
}
impl Default for X86GCCustomStrategy {
fn default() -> Self {
X86GCCustomStrategy {
name: "custom".to_string(),
uses_statepoints: false,
uses_shadow_stack: false,
supports_generational: false,
requires_write_barriers: false,
requires_read_barriers: false,
supports_concurrent_marking: false,
suspension: X86GCSuspension::default(),
safepoint_frequency: X86GCSafepointFrequency::default(),
write_barrier_kind: X86GCWriteBarrierKind::None,
read_barrier_kind: X86GCReadBarrierKind::None,
params: Vec::new(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86GCSuspension {
None,
Polling,
SignalBased,
PageFault,
Hybrid,
}
impl Default for X86GCSuspension {
fn default() -> Self {
X86GCSuspension::Polling
}
}
impl fmt::Display for X86GCSuspension {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCSuspension::None => write!(f, "none"),
X86GCSuspension::Polling => write!(f, "polling"),
X86GCSuspension::SignalBased => write!(f, "signal"),
X86GCSuspension::PageFault => write!(f, "pagefault"),
X86GCSuspension::Hybrid => write!(f, "hybrid"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86GCSafepointFrequency {
Always,
EveryNthBackedge(u32),
EntryAndCalls,
EntryOnly,
StatepointsOnly,
}
impl Default for X86GCSafepointFrequency {
fn default() -> Self {
X86GCSafepointFrequency::Always
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86GCWriteBarrierKind {
None,
CardMarking,
RememberedSet,
SATB,
Yuasa,
Steele,
}
impl fmt::Display for X86GCWriteBarrierKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCWriteBarrierKind::None => write!(f, "none"),
X86GCWriteBarrierKind::CardMarking => write!(f, "card-marking"),
X86GCWriteBarrierKind::RememberedSet => write!(f, "remembered-set"),
X86GCWriteBarrierKind::SATB => write!(f, "satb"),
X86GCWriteBarrierKind::Yuasa => write!(f, "yuasa"),
X86GCWriteBarrierKind::Steele => write!(f, "steele"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86GCReadBarrierKind {
None,
Brooks,
SelfHealing,
Conditional,
AcquireLoad,
}
impl fmt::Display for X86GCReadBarrierKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCReadBarrierKind::None => write!(f, "none"),
X86GCReadBarrierKind::Brooks => write!(f, "brooks"),
X86GCReadBarrierKind::SelfHealing => write!(f, "self-healing"),
X86GCReadBarrierKind::Conditional => write!(f, "conditional"),
X86GCReadBarrierKind::AcquireLoad => write!(f, "acquire-load"),
}
}
}
impl X86GCStrategy {
pub fn name(&self) -> &str {
match self {
X86GCStrategy::ShadowStack => "shadow-stack",
X86GCStrategy::Statepoint => "statepoint",
X86GCStrategy::CoreCLR => "coreclr",
X86GCStrategy::Erlang => "erlang",
X86GCStrategy::OCaml => "ocaml",
X86GCStrategy::Custom(c) => c.name.as_str(),
}
}
pub fn uses_statepoints(&self) -> bool {
match self {
X86GCStrategy::Statepoint => true,
X86GCStrategy::Custom(c) => c.uses_statepoints,
_ => false,
}
}
pub fn uses_shadow_stack(&self) -> bool {
match self {
X86GCStrategy::ShadowStack => true,
X86GCStrategy::Custom(c) => c.uses_shadow_stack,
_ => false,
}
}
pub fn supports_generational(&self) -> bool {
match self {
X86GCStrategy::ShadowStack | X86GCStrategy::Statepoint | X86GCStrategy::CoreCLR => true,
X86GCStrategy::Custom(c) => c.supports_generational,
_ => false,
}
}
pub fn requires_write_barriers(&self) -> bool {
match self {
X86GCStrategy::CoreCLR => true,
X86GCStrategy::Custom(c) => {
c.requires_write_barriers && c.write_barrier_kind != X86GCWriteBarrierKind::None
}
_ => false,
}
}
pub fn requires_read_barriers(&self) -> bool {
match self {
X86GCStrategy::Custom(c) => {
c.requires_read_barriers && c.read_barrier_kind != X86GCReadBarrierKind::None
}
_ => false,
}
}
pub fn supports_concurrent_marking(&self) -> bool {
match self {
X86GCStrategy::ShadowStack | X86GCStrategy::Statepoint | X86GCStrategy::CoreCLR => true,
X86GCStrategy::Custom(c) => c.supports_concurrent_marking,
_ => false,
}
}
pub fn uses_cooperative_suspension(&self) -> bool {
match self {
X86GCStrategy::CoreCLR | X86GCStrategy::Erlang => true,
X86GCStrategy::Custom(c) => c.suspension != X86GCSuspension::None,
_ => false,
}
}
pub fn safepoint_frequency(&self) -> X86GCSafepointFrequency {
match self {
X86GCStrategy::ShadowStack => X86GCSafepointFrequency::StatepointsOnly,
X86GCStrategy::Statepoint => X86GCSafepointFrequency::StatepointsOnly,
X86GCStrategy::CoreCLR => X86GCSafepointFrequency::Always,
X86GCStrategy::Erlang => X86GCSafepointFrequency::EveryNthBackedge(1),
X86GCStrategy::OCaml => X86GCSafepointFrequency::EntryAndCalls,
X86GCStrategy::Custom(c) => c.safepoint_frequency,
}
}
pub fn write_barrier_kind(&self) -> X86GCWriteBarrierKind {
match self {
X86GCStrategy::CoreCLR => X86GCWriteBarrierKind::CardMarking,
X86GCStrategy::Custom(c) => c.write_barrier_kind,
_ => X86GCWriteBarrierKind::None,
}
}
pub fn read_barrier_kind(&self) -> X86GCReadBarrierKind {
match self {
X86GCStrategy::Custom(c) => c.read_barrier_kind,
_ => X86GCReadBarrierKind::None,
}
}
pub fn suspension(&self) -> X86GCSuspension {
match self {
X86GCStrategy::CoreCLR => X86GCSuspension::Polling,
X86GCStrategy::Erlang => X86GCSuspension::None,
X86GCStrategy::Custom(c) => c.suspension,
_ => X86GCSuspension::None,
}
}
}
impl Default for X86GCStrategy {
fn default() -> Self {
X86GCStrategy::Statepoint
}
}
impl fmt::Display for X86GCStrategy {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.name())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct X86GCRoot {
pub id: u32,
pub frame_offset: i32,
pub register: Option<u16>,
pub pointer_size: u8,
pub type_metadata: Option<X86GCRootType>,
pub is_derived: bool,
pub base_root_id: Option<u32>,
pub interior_offset: i64,
pub is_constant: bool,
pub needs_relocation: bool,
pub source_language: X86GCRootLanguage,
pub is_live: bool,
pub is_callee_saved: bool,
pub live_start_offset: u32,
pub live_end_offset: u32,
pub symbol_name: Option<String>,
pub tls_index: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum X86GCRootType {
Pointer,
Struct {
size: u32,
pointer_bitmap: Vec<u64>,
},
PointerArray {
length: u32,
},
StructArray {
length: u32,
struct_type: Box<X86GCRootType>,
},
TaggedUnion {
tag_offset: u32,
variants: HashMap<u32, Box<X86GCRootType>>,
},
Opaque {
size: u32,
},
Leaf,
WeakPointer,
Finalizable,
Custom(String),
}
impl X86GCRootType {
pub fn has_pointers(&self) -> bool {
match self {
X86GCRootType::Pointer => true,
X86GCRootType::Struct { pointer_bitmap, .. } => pointer_bitmap.iter().any(|&w| w != 0),
X86GCRootType::PointerArray { .. } => true,
X86GCRootType::StructArray { struct_type, .. } => struct_type.has_pointers(),
X86GCRootType::TaggedUnion { .. } => true,
X86GCRootType::Opaque { .. } => true, X86GCRootType::Leaf => false,
X86GCRootType::WeakPointer => true,
X86GCRootType::Finalizable => true,
X86GCRootType::Custom(_) => true,
}
}
pub fn is_precise(&self) -> bool {
!matches!(self, X86GCRootType::Opaque { .. })
}
pub fn size(&self) -> u32 {
match self {
X86GCRootType::Pointer => 8,
X86GCRootType::Struct { size, .. } => *size,
X86GCRootType::PointerArray { length } => length * 8,
X86GCRootType::StructArray {
length,
struct_type,
..
} => length * struct_type.size(),
X86GCRootType::TaggedUnion { .. } => 16, X86GCRootType::Opaque { size } => *size,
X86GCRootType::Leaf => 0,
X86GCRootType::WeakPointer => 8,
X86GCRootType::Finalizable => 8,
X86GCRootType::Custom(_) => 8,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86GCRootLanguage {
C,
Rust,
Java,
CSharp,
Go,
Erlang,
OCaml,
Python,
JavaScript,
Unknown,
}
impl Default for X86GCRootLanguage {
fn default() -> Self {
X86GCRootLanguage::Unknown
}
}
impl fmt::Display for X86GCRootLanguage {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCRootLanguage::C => write!(f, "c"),
X86GCRootLanguage::Rust => write!(f, "rust"),
X86GCRootLanguage::Java => write!(f, "java"),
X86GCRootLanguage::CSharp => write!(f, "csharp"),
X86GCRootLanguage::Go => write!(f, "go"),
X86GCRootLanguage::Erlang => write!(f, "erlang"),
X86GCRootLanguage::OCaml => write!(f, "ocaml"),
X86GCRootLanguage::Python => write!(f, "python"),
X86GCRootLanguage::JavaScript => write!(f, "javascript"),
X86GCRootLanguage::Unknown => write!(f, "unknown"),
}
}
}
impl X86GCRoot {
pub fn new_base(id: u32, frame_offset: i32) -> Self {
X86GCRoot {
id,
frame_offset,
register: None,
pointer_size: 8,
type_metadata: Some(X86GCRootType::Pointer),
is_derived: false,
base_root_id: None,
interior_offset: 0,
is_constant: false,
needs_relocation: true,
source_language: X86GCRootLanguage::Unknown,
is_live: true,
is_callee_saved: false,
live_start_offset: 0,
live_end_offset: u32::MAX,
symbol_name: None,
tls_index: None,
}
}
pub fn new_derived(
id: u32,
frame_offset: i32,
base_root_id: u32,
interior_offset: i64,
) -> Self {
X86GCRoot {
id,
frame_offset,
register: None,
pointer_size: 8,
type_metadata: Some(X86GCRootType::Pointer),
is_derived: true,
base_root_id: Some(base_root_id),
interior_offset,
is_constant: false,
needs_relocation: true,
source_language: X86GCRootLanguage::Unknown,
is_live: true,
is_callee_saved: false,
live_start_offset: 0,
live_end_offset: u32::MAX,
symbol_name: None,
tls_index: None,
}
}
pub fn new_register(id: u32, dwarf_reg: u16) -> Self {
X86GCRoot {
id,
frame_offset: 0,
register: Some(dwarf_reg),
pointer_size: 8,
type_metadata: Some(X86GCRootType::Pointer),
is_derived: false,
base_root_id: None,
interior_offset: 0,
is_constant: false,
needs_relocation: true,
source_language: X86GCRootLanguage::Unknown,
is_live: true,
is_callee_saved: false,
live_start_offset: 0,
live_end_offset: u32::MAX,
symbol_name: None,
tls_index: None,
}
}
pub fn new_static(id: u32, symbol_name: String) -> Self {
X86GCRoot {
id,
frame_offset: 0,
register: None,
pointer_size: 8,
type_metadata: Some(X86GCRootType::Pointer),
is_derived: false,
base_root_id: None,
interior_offset: 0,
is_constant: true,
needs_relocation: true,
source_language: X86GCRootLanguage::Unknown,
is_live: true,
is_callee_saved: false,
live_start_offset: 0,
live_end_offset: u32::MAX,
symbol_name: Some(symbol_name),
tls_index: None,
}
}
pub fn new_thread_local(id: u32, tls_index: u32) -> Self {
X86GCRoot {
id,
frame_offset: 0,
register: None,
pointer_size: 8,
type_metadata: Some(X86GCRootType::Pointer),
is_derived: false,
base_root_id: None,
interior_offset: 0,
is_constant: false,
needs_relocation: true,
source_language: X86GCRootLanguage::Unknown,
is_live: true,
is_callee_saved: false,
live_start_offset: 0,
live_end_offset: u32::MAX,
symbol_name: None,
tls_index: Some(tls_index),
}
}
pub fn with_type_metadata(mut self, metadata: X86GCRootType) -> Self {
self.type_metadata = Some(metadata);
self
}
pub fn with_language(mut self, language: X86GCRootLanguage) -> Self {
self.source_language = language;
self
}
pub fn mark_callee_saved(mut self) -> Self {
self.is_callee_saved = true;
self
}
pub fn with_live_range(mut self, start: u32, end: u32) -> Self {
self.live_start_offset = start;
self.live_end_offset = end;
self
}
pub fn mark_dead(mut self) -> Self {
self.is_live = false;
self
}
pub fn slot_size(&self) -> u32 {
self.type_metadata
.as_ref()
.map(|t| t.size())
.unwrap_or(self.pointer_size as u32)
}
pub fn is_live_at(&self, offset: u32) -> bool {
self.is_live && offset >= self.live_start_offset && offset <= self.live_end_offset
}
pub fn base_root_id(&self) -> Option<u32> {
self.base_root_id
}
pub fn needs_conservative_scan(&self) -> bool {
self.type_metadata
.as_ref()
.map(|t| !t.is_precise())
.unwrap_or(true)
}
}
#[derive(Debug, Clone)]
pub struct X86GCRootCandidate {
pub raw_value: u64,
pub stack_address: u64,
pub frame_offset: i32,
pub is_valid_heap_ptr: bool,
pub scan_mode: X86GCScanMode,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCScanMode {
Precise,
Conservative,
Hybrid,
}
#[derive(Debug, Clone)]
pub struct X86GCBarrier {
pub barrier_type: X86GCBarrierType,
pub write_barrier_kind: X86GCWriteBarrierKind,
pub read_barrier_kind: X86GCReadBarrierKind,
pub object_base: u64,
pub field_offset: u64,
pub new_value: Option<u64>,
pub old_value: Option<u64>,
pub is_array_element: bool,
pub emit_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCBarrierType {
Write,
Read,
ReadWrite,
CAS,
AtomicRMW,
}
impl X86GCBarrier {
pub fn new_card_marking(object_base: u64, field_offset: u64, new_value: u64) -> Self {
X86GCBarrier {
barrier_type: X86GCBarrierType::Write,
write_barrier_kind: X86GCWriteBarrierKind::CardMarking,
read_barrier_kind: X86GCReadBarrierKind::None,
object_base,
field_offset,
new_value: Some(new_value),
old_value: None,
is_array_element: false,
emit_count: 0,
}
}
pub fn new_satb(object_base: u64, field_offset: u64, old_value: u64) -> Self {
X86GCBarrier {
barrier_type: X86GCBarrierType::Write,
write_barrier_kind: X86GCWriteBarrierKind::SATB,
read_barrier_kind: X86GCReadBarrierKind::None,
object_base,
field_offset,
new_value: None,
old_value: Some(old_value),
is_array_element: false,
emit_count: 0,
}
}
pub fn new_generational(object_base: u64, field_offset: u64, new_value: u64) -> Self {
X86GCBarrier {
barrier_type: X86GCBarrierType::Write,
write_barrier_kind: X86GCWriteBarrierKind::CardMarking,
read_barrier_kind: X86GCReadBarrierKind::None,
object_base,
field_offset,
new_value: Some(new_value),
old_value: None,
is_array_element: false,
emit_count: 0,
}
}
pub fn new_brooks_read(object_base: u64) -> Self {
X86GCBarrier {
barrier_type: X86GCBarrierType::Read,
write_barrier_kind: X86GCWriteBarrierKind::None,
read_barrier_kind: X86GCReadBarrierKind::Brooks,
object_base,
field_offset: 0,
new_value: None,
old_value: None,
is_array_element: false,
emit_count: 0,
}
}
pub fn new_acquire_load(object_base: u64) -> Self {
X86GCBarrier {
barrier_type: X86GCBarrierType::Read,
write_barrier_kind: X86GCWriteBarrierKind::None,
read_barrier_kind: X86GCReadBarrierKind::AcquireLoad,
object_base,
field_offset: 0,
new_value: None,
old_value: None,
is_array_element: false,
emit_count: 0,
}
}
pub fn mark_array_element(mut self) -> Self {
self.is_array_element = true;
self
}
pub fn record_emit(&mut self) {
self.emit_count += 1;
}
pub fn emit_x86_card_marking_asm(&self, card_table_base: u64, card_shift: u8) -> Vec<u8> {
let new_val = self.new_value.unwrap_or(0);
let mut code = Vec::with_capacity(32);
let full_addr = self.object_base.wrapping_add(self.field_offset);
code.push(0x48); code.push(0xC7); if (full_addr >> 32) == 0 && full_addr <= 0x7FFF_FFFF {
code.push(0x04);
code.push(0x25);
code.extend_from_slice(&(full_addr as u32).to_le_bytes());
} else {
code.push(0x04); code.push(0x24); code.push(0x00);
}
code.extend_from_slice(&(new_val as u32).to_le_bytes());
code.push(0x48); code.push(0xB8); code.extend_from_slice(&self.object_base.to_le_bytes());
code.push(0x48); code.push(0xC1); code.push(0xE8); code.push(card_shift);
code.push(0xC6); code.push(0x80); code.extend_from_slice(&(card_table_base as u32).to_le_bytes());
code.push(0x00);
code
}
pub fn emit_x86_satb_asm(&self, satb_buffer_addr: u64, satb_index_addr: u64) -> Vec<u8> {
let mut code = Vec::with_capacity(48);
code.push(0x48);
code.push(0xA1);
code.extend_from_slice(&satb_index_addr.to_le_bytes());
let old = self.old_value.unwrap_or(0);
code.push(0x48);
code.push(0xBA);
code.extend_from_slice(&old.to_le_bytes());
code.push(0x48);
code.push(0x89);
code.push(0xD4);
code.push(0xC0);
code.extend_from_slice(&satb_buffer_addr.to_le_bytes());
code.push(0x48);
code.push(0xFF);
code.push(0xC0);
code.push(0x48);
code.push(0x89);
code.push(0x04);
code.push(0x25);
code.extend_from_slice(&satb_index_addr.to_le_bytes());
code
}
pub fn emit_x86_brooks_read_asm(&self) -> Vec<u8> {
let mut code = Vec::with_capacity(16);
let full = self.object_base;
code.push(0x48);
code.push(0xA1);
code.extend_from_slice(&full.to_le_bytes());
code.push(0x48);
code.push(0x8B);
code.push(0x80);
code.extend_from_slice(&(self.field_offset as u32).to_le_bytes());
code
}
pub fn emit_x86_acquire_load_asm(&self) -> Vec<u8> {
let mut code = Vec::with_capacity(24);
code.push(0x48);
code.push(0xA1);
code.extend_from_slice(&self.object_base.to_le_bytes());
code.push(0x0F);
code.push(0xAE);
code.push(0xE8);
code
}
}
#[derive(Debug, Clone)]
pub struct X86GCBumpAllocator {
pub base: u64,
pub cursor: u64,
pub limit: u64,
pub bytes_allocated: u64,
pub allocation_count: u64,
pub slow_path_count: u64,
pub alignment: u32,
}
impl X86GCBumpAllocator {
pub fn new(base: u64, size: u64, alignment: u32) -> Self {
X86GCBumpAllocator {
base,
cursor: base,
limit: base + size,
bytes_allocated: 0,
allocation_count: 0,
slow_path_count: 0,
alignment,
}
}
pub fn allocate_fast(&mut self, size: u64) -> u64 {
let aligned_size = (size + self.alignment as u64 - 1) & !(self.alignment as u64 - 1);
let new_cursor = self.cursor.wrapping_add(aligned_size);
if new_cursor <= self.limit {
let result = self.cursor;
self.cursor = new_cursor;
self.bytes_allocated += aligned_size;
self.allocation_count += 1;
result
} else {
self.slow_path_count += 1;
0
}
}
pub fn reset(&mut self) {
self.cursor = self.base;
}
pub fn remaining(&self) -> u64 {
self.limit.saturating_sub(self.cursor)
}
pub fn is_exhausted(&self) -> bool {
self.cursor >= self.limit
}
pub fn emit_x86_fast_path_asm(&self, size: u64, cursor_addr: u64, limit_addr: u64) -> Vec<u8> {
let mut code = Vec::with_capacity(32);
code.push(0x48);
code.push(0xA1);
code.extend_from_slice(&cursor_addr.to_le_bytes());
let aligned = (size + self.alignment as u64 - 1) & !(self.alignment as u64 - 1);
code.push(0x48);
if aligned <= 0x7FFF_FFFF {
code.push(0x8D);
code.push(0x90);
code.extend_from_slice(&(aligned as u32).to_le_bytes());
} else {
code.push(0x8D); code.push(0x90);
code.extend_from_slice(&(aligned as u32).to_le_bytes());
}
code.push(0x48);
code.push(0x3B);
code.push(0x14);
code.push(0x25);
code.extend_from_slice(&limit_addr.to_le_bytes());
code.push(0x0F);
code.push(0x87);
code.push(0x06);
code.push(0x00);
code.push(0x00);
code.push(0x00);
code.push(0x48);
code.push(0x89);
code.push(0x14);
code.push(0x25);
code.extend_from_slice(&cursor_addr.to_le_bytes());
code
}
}
#[derive(Debug, Clone)]
pub struct X86GCFreeListAllocator {
pub free_lists: BTreeMap<u32, Vec<X86GCFreeBlock>>,
pub coalesce: bool,
pub bytes_allocated: u64,
pub bytes_freed: u64,
pub max_heap_size: u64,
pub current_heap_size: u64,
pub allocation_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCFreeBlock {
pub address: u64,
pub size: u32,
}
impl X86GCFreeListAllocator {
pub fn new(max_heap_size: u64) -> Self {
X86GCFreeListAllocator {
free_lists: BTreeMap::new(),
coalesce: true,
bytes_allocated: 0,
bytes_freed: 0,
max_heap_size,
current_heap_size: 0,
allocation_count: 0,
}
}
pub fn add_free_block(&mut self, address: u64, size: u32) {
let size_class = Self::size_class(size);
self.free_lists
.entry(size_class)
.or_insert_with(Vec::new)
.push(X86GCFreeBlock { address, size });
}
pub fn allocate(&mut self, size: u32) -> u64 {
let size_class = Self::size_class(size);
if let Some(blocks) = self.free_lists.get_mut(&size_class) {
if let Some((idx, _)) = blocks.iter().enumerate().find(|(_, b)| b.size >= size) {
let block = blocks.remove(idx);
self.bytes_allocated += block.size as u64;
self.allocation_count += 1;
if block.size > size {
let remainder_addr = block.address + size as u64;
let remainder_size = block.size - size;
self.add_free_block(remainder_addr, remainder_size);
}
return block.address;
}
}
for (&sc, blocks) in self.free_lists.range_mut(size_class + 1..) {
if sc < size_class {
continue;
}
if let Some(block) = blocks.pop() {
if block.size >= size {
self.bytes_allocated += block.size as u64;
self.allocation_count += 1;
if block.size > size {
self.add_free_block(block.address + size as u64, block.size - size);
}
return block.address;
}
blocks.push(block);
}
}
0
}
pub fn free(&mut self, address: u64, size: u32) {
self.bytes_freed += size as u64;
self.add_free_block(address, size);
if self.coalesce {
self.try_coalesce(address);
}
}
fn try_coalesce(&mut self, _address: u64) {
let mut all_blocks: Vec<X86GCFreeBlock> = self
.free_lists
.values()
.flat_map(|v| v.iter().copied())
.collect();
all_blocks.sort_by_key(|b| b.address);
let mut merged: Vec<X86GCFreeBlock> = Vec::new();
for block in all_blocks {
if let Some(last) = merged.last_mut() {
if last.address + last.size as u64 == block.address {
last.size += block.size;
continue;
}
}
merged.push(block);
}
self.free_lists.clear();
for block in merged {
self.add_free_block(block.address, block.size);
}
}
fn size_class(size: u32) -> u32 {
if size <= 16 {
16
} else if size <= 32 {
32
} else if size <= 64 {
64
} else if size <= 128 {
128
} else if size <= 256 {
256
} else if size <= 512 {
512
} else if size <= 1024 {
1024
} else if size <= 2048 {
2048
} else if size <= 4096 {
4096
} else if size <= 8192 {
8192
} else {
((size + 85 * 1024 - 1) / (85 * 1024)) * 85 * 1024
}
}
pub fn free_bytes(&self) -> u64 {
self.free_lists
.values()
.flat_map(|v| v.iter().map(|b| b.size as u64))
.sum()
}
pub fn fragmentation_ratio(&self) -> f64 {
let total_free = self.free_bytes();
let free_blocks: usize = self.free_lists.values().map(|v| v.len()).sum();
if free_blocks == 0 {
return 0.0;
}
let avg_block_size = total_free as f64 / free_blocks as f64;
1.0 - (avg_block_size / avg_block_size.max(1.0))
}
}
#[derive(Debug, Clone)]
pub struct X86GCTLAB {
pub thread_id: u32,
pub base: u64,
pub cursor: Arc<Mutex<u64>>,
pub limit: u64,
pub refill_count: u64,
pub bytes_allocated: u64,
pub is_active: bool,
}
impl X86GCTLAB {
pub fn new(thread_id: u32, base: u64, size: u64) -> Self {
X86GCTLAB {
thread_id,
base,
cursor: Arc::new(Mutex::new(base)),
limit: base + size,
refill_count: 0,
bytes_allocated: 0,
is_active: true,
}
}
pub fn allocate_fast(&mut self, size: u64) -> u64 {
let aligned = (size + 7) & !7;
let mut cursor = self.cursor.lock().unwrap();
let new_cursor = (*cursor).wrapping_add(aligned);
if new_cursor <= self.limit {
let result = *cursor;
*cursor = new_cursor;
self.bytes_allocated += aligned;
result
} else {
0 }
}
pub fn refill(&mut self, new_base: u64, new_size: u64) {
self.base = new_base;
*self.cursor.lock().unwrap() = new_base;
self.limit = new_base + new_size;
self.refill_count += 1;
self.is_active = true;
}
pub fn retire(&mut self) {
self.is_active = false;
}
pub fn remaining(&self) -> u64 {
let cursor = *self.cursor.lock().unwrap();
self.limit.saturating_sub(cursor)
}
pub fn needs_refill(&self, size: u64) -> bool {
self.remaining() < size
}
}
#[derive(Debug, Clone)]
pub struct X86GCLargeObjectAllocator {
pub allocations: HashMap<u64, u64>,
pub total_bytes: u64,
pub count: u64,
pub threshold: u64,
}
impl X86GCLargeObjectAllocator {
pub fn new(threshold: u64) -> Self {
X86GCLargeObjectAllocator {
allocations: HashMap::new(),
total_bytes: 0,
count: 0,
threshold,
}
}
pub fn allocate(&mut self, size: u64) -> u64 {
if size < self.threshold {
return 0; }
let addr = 0x7F00_0000_0000u64 + self.total_bytes;
self.allocations.insert(addr, size);
self.total_bytes += size;
self.count += 1;
addr
}
pub fn free(&mut self, address: u64) -> bool {
if let Some(size) = self.allocations.remove(&address) {
self.total_bytes -= size;
self.count -= 1;
true
} else {
false
}
}
pub fn contains(&self, address: u64) -> bool {
self.allocations
.iter()
.any(|(&base, &size)| address >= base && address < base + size)
}
pub fn find_enclosing_object(&self, address: u64) -> Option<(u64, u64)> {
self.allocations
.iter()
.find(|&(&base, &size)| address >= base && address < base + size)
.map(|(&base, &size)| (base, size))
}
}
#[derive(Debug, Clone)]
pub struct X86GCMarkSweepCollector {
pub heap_base: u64,
pub heap_size: u64,
pub free_list: Vec<X86GCCollectorFreeBlock>,
pub mark_bits: Vec<u64>,
pub object_sizes: HashMap<u64, u64>,
pub object_count: u64,
pub bytes_allocated: u64,
pub bytes_freed_last: u64,
pub collections: u64,
pub mark_stack: Vec<u64>,
pub collecting: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCCollectorFreeBlock {
pub address: u64,
pub size: u64,
}
impl X86GCMarkSweepCollector {
pub fn new(heap_base: u64, heap_size: u64) -> Self {
let allocation_units = (heap_size / X86_GC_OBJECT_ALIGNMENT as u64) as usize;
let mark_words = (allocation_units + 63) / 64;
X86GCMarkSweepCollector {
heap_base,
heap_size,
free_list: Vec::new(),
mark_bits: vec![0; mark_words],
object_sizes: HashMap::new(),
object_count: 0,
bytes_allocated: 0,
bytes_freed_last: 0,
collections: 0,
mark_stack: Vec::new(),
collecting: false,
}
}
pub fn allocate(&mut self, size: u64) -> u64 {
let aligned_size =
(size + X86_GC_OBJECT_ALIGNMENT as u64 - 1) & !(X86_GC_OBJECT_ALIGNMENT as u64 - 1);
if let Some(pos) = self.free_list.iter().position(|b| b.size >= aligned_size) {
let block = self.free_list.remove(pos);
if block.size > aligned_size {
self.free_list.push(X86GCCollectorFreeBlock {
address: block.address + aligned_size,
size: block.size - aligned_size,
});
}
self.object_sizes.insert(block.address, aligned_size);
self.object_count += 1;
self.bytes_allocated += aligned_size;
return block.address;
}
0
}
pub fn set_mark(&mut self, address: u64) {
let offset = address.saturating_sub(self.heap_base);
if offset < self.heap_size {
let unit_index = (offset / X86_GC_OBJECT_ALIGNMENT as u64) as usize;
let word_idx = unit_index / 64;
let bit_idx = unit_index % 64;
if word_idx < self.mark_bits.len() {
self.mark_bits[word_idx] |= 1u64 << bit_idx;
}
}
}
pub fn is_marked(&self, address: u64) -> bool {
let offset = address.saturating_sub(self.heap_base);
if offset < self.heap_size {
let unit_index = (offset / X86_GC_OBJECT_ALIGNMENT as u64) as usize;
let word_idx = unit_index / 64;
let bit_idx = unit_index % 64;
if word_idx < self.mark_bits.len() {
return (self.mark_bits[word_idx] & (1u64 << bit_idx)) != 0;
}
}
false
}
pub fn mark_phase(&mut self, roots: &[X86GCRoot]) {
self.collecting = true;
for root in roots {
if root.is_live {
self.mark_stack.push(self.heap_base); }
}
while let Some(obj_addr) = self.mark_stack.pop() {
if self.is_marked(obj_addr) {
continue;
}
self.set_mark(obj_addr);
if let Some(&obj_size) = self.object_sizes.get(&obj_addr) {
let num_fields = obj_size / 8;
for i in 0..num_fields {
let field_addr = obj_addr + i * 8;
let _ = field_addr;
}
}
}
self.collecting = false;
self.collections += 1;
}
pub fn sweep_phase(&mut self) -> u64 {
let mut freed_bytes = 0u64;
let mut to_free: Vec<u64> = Vec::new();
for (&addr, &size) in &self.object_sizes {
if !self.is_marked(addr) {
self.free_list.push(X86GCCollectorFreeBlock {
address: addr,
size,
});
freed_bytes += size;
to_free.push(addr);
}
}
let to_free_len = to_free.len() as u64;
for addr in to_free {
self.object_sizes.remove(&addr);
}
for word in &mut self.mark_bits {
*word = 0;
}
self.bytes_freed_last = freed_bytes;
self.object_count -= to_free_len;
freed_bytes
}
pub fn collect(&mut self, roots: &[X86GCRoot]) -> u64 {
self.mark_phase(roots);
self.sweep_phase()
}
}
#[derive(Debug, Clone)]
pub struct X86GCCopyingCollector {
pub semispace_size: u64,
pub fromspace_base: u64,
pub tospace_base: u64,
pub free: u64,
pub scan: u64,
pub collecting: bool,
pub bytes_allocated: u64,
pub bytes_copied_last: u64,
pub collections: u64,
pub forwarding: HashMap<u64, u64>,
}
impl X86GCCopyingCollector {
pub fn new(fromspace_base: u64, tospace_base: u64, semispace_size: u64) -> Self {
X86GCCopyingCollector {
semispace_size,
fromspace_base,
tospace_base,
free: fromspace_base, scan: fromspace_base,
collecting: false,
bytes_allocated: 0,
bytes_copied_last: 0,
collections: 0,
forwarding: HashMap::new(),
}
}
pub fn allocate_fast(&mut self, size: u64) -> u64 {
let aligned = (size + 7) & !7;
let new_free = self.free.wrapping_add(aligned);
if new_free <= self.fromspace_base + self.semispace_size {
let result = self.free;
self.free = new_free;
self.bytes_allocated += aligned;
result
} else {
0 }
}
pub fn forward(&mut self, addr: u64, size: u64) -> u64 {
if let Some(&new_addr) = self.forwarding.get(&addr) {
return new_addr;
}
let aligned = (size + 7) & !7;
let new_addr = self.free;
self.free = self.free.wrapping_add(aligned);
self.forwarding.insert(addr, new_addr);
self.bytes_copied_last += aligned;
new_addr
}
pub fn cheney_scan(&mut self, roots: &[u64]) {
self.scan = self.tospace_base;
for &root_addr in roots {
let new_addr = self.forward(root_addr, 64); let _ = new_addr;
}
while self.scan < self.free {
self.scan += 64; }
}
pub fn collect(&mut self, roots: &[u64]) -> u64 {
self.collecting = true;
self.bytes_copied_last = 0;
std::mem::swap(&mut self.fromspace_base, &mut self.tospace_base);
self.free = self.tospace_base;
self.forwarding.clear();
self.cheney_scan(roots);
self.collecting = false;
self.collections += 1;
self.bytes_copied_last
}
}
#[derive(Debug, Clone)]
pub struct X86GCGenerationalCollector {
pub nursery_base: u64,
pub nursery_size: u64,
pub nursery_free: u64,
pub remembered_set: HashSet<u64>,
pub old_collector: X86GCMarkSweepCollector,
pub minor_collections: u64,
pub major_collections: u64,
pub promoted_last: u64,
pub promote_threshold: u32,
pub object_ages: HashMap<u64, u32>,
pub total_bytes_promoted: u64,
}
impl X86GCGenerationalCollector {
pub fn new(
nursery_base: u64,
nursery_size: u64,
old_heap_base: u64,
old_heap_size: u64,
) -> Self {
X86GCGenerationalCollector {
nursery_base,
nursery_size,
nursery_free: nursery_base,
remembered_set: HashSet::new(),
old_collector: X86GCMarkSweepCollector::new(old_heap_base, old_heap_size),
minor_collections: 0,
major_collections: 0,
promoted_last: 0,
promote_threshold: 3,
object_ages: HashMap::new(),
total_bytes_promoted: 0,
}
}
pub fn nursery_allocate(&mut self, size: u64) -> u64 {
let aligned = (size + 7) & !7;
let new_free = self.nursery_free.wrapping_add(aligned);
if new_free <= self.nursery_base + self.nursery_size {
let result = self.nursery_free;
self.nursery_free = new_free;
result
} else {
0 }
}
pub fn record_write(&mut self, old_gen_address: u64) {
if old_gen_address < self.nursery_base
|| old_gen_address >= self.nursery_base + self.nursery_size
{
self.remembered_set.insert(old_gen_address);
}
}
pub fn minor_collect(&mut self, roots: &[X86GCRoot], _root_values: &[u64]) {
self.minor_collections += 1;
self.promoted_last = 0;
for (&old_obj, age) in self.object_ages.iter_mut() {
if old_obj < self.nursery_base || old_obj >= self.nursery_base + self.nursery_size {
*age += 1;
if *age >= self.promote_threshold {
let _obj_size = 64u64; self.total_bytes_promoted += 64;
self.promoted_last += 1;
}
}
}
self.nursery_free = self.nursery_base;
}
pub fn major_collect(&mut self, roots: &[X86GCRoot]) -> u64 {
self.major_collections += 1;
self.old_collector.collect(roots)
}
pub fn collect(&mut self, roots: &[X86GCRoot], root_values: &[u64]) {
self.minor_collect(roots, root_values);
let occupancy =
self.old_collector.bytes_allocated as f64 / self.old_collector.heap_size as f64;
if occupancy > X86_GC_MIN_HEAP_OCCUPANCY {
self.major_collect(roots);
}
}
}
#[derive(Debug, Clone)]
pub struct X86GCIncrementalCollector {
pub mark_bytes: Vec<u8>,
pub gray_stack: Vec<u64>,
pub marking: bool,
pub sweeping: bool,
pub sweep_cursor: u64,
pub heap_base: u64,
pub heap_size: u64,
pub collections: u64,
pub marking_steps: u64,
pub objects_marked: u64,
pub write_barriers_applied: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCTriColor {
White = 0,
Gray = 1,
Black = 2,
}
impl X86GCIncrementalCollector {
pub fn new(heap_base: u64, heap_size: u64) -> Self {
let units = (heap_size / X86_GC_OBJECT_ALIGNMENT as u64) as usize;
X86GCIncrementalCollector {
mark_bytes: vec![X86GCTriColor::White as u8; units],
gray_stack: Vec::new(),
marking: false,
sweeping: false,
sweep_cursor: heap_base,
heap_base,
heap_size,
collections: 0,
marking_steps: 0,
objects_marked: 0,
write_barriers_applied: 0,
}
}
pub fn color(&self, addr: u64) -> X86GCTriColor {
let offset = addr.saturating_sub(self.heap_base);
if offset < self.heap_size {
let unit = (offset / X86_GC_OBJECT_ALIGNMENT as u64) as usize;
if unit < self.mark_bytes.len() {
return match self.mark_bytes[unit] {
0 => X86GCTriColor::White,
1 => X86GCTriColor::Gray,
_ => X86GCTriColor::Black,
};
}
}
X86GCTriColor::White
}
pub fn set_color(&mut self, addr: u64, color: X86GCTriColor) {
let offset = addr.saturating_sub(self.heap_base);
if offset < self.heap_size {
let unit = (offset / X86_GC_OBJECT_ALIGNMENT as u64) as usize;
if unit < self.mark_bytes.len() {
self.mark_bytes[unit] = color as u8;
}
}
}
pub fn start_mark(&mut self, roots: &[u64]) {
self.marking = true;
self.sweeping = false;
for byte in &mut self.mark_bytes {
*byte = X86GCTriColor::White as u8;
}
for &root_ptr in roots {
self.set_color(root_ptr, X86GCTriColor::Gray);
self.gray_stack.push(root_ptr);
}
}
pub fn marking_step(&mut self, _scan_n: usize) -> bool {
if !self.marking {
return true;
}
self.marking_steps += 1;
for _ in 0..std::cmp::min(10, self.gray_stack.len()) {
if let Some(obj) = self.gray_stack.pop() {
self.set_color(obj, X86GCTriColor::Black);
self.objects_marked += 1;
}
}
if self.gray_stack.is_empty() {
self.marking = false;
self.sweeping = true;
self.sweep_cursor = self.heap_base;
self.collections += 1;
true
} else {
false
}
}
pub fn write_barrier(&mut self, black_obj: u64, white_obj: u64) {
if self.marking
&& self.color(black_obj) == X86GCTriColor::Black
&& self.color(white_obj) == X86GCTriColor::White
{
self.set_color(white_obj, X86GCTriColor::Gray);
self.gray_stack.push(white_obj);
self.write_barriers_applied += 1;
}
}
pub fn sweeping_step(&mut self) -> bool {
if !self.sweeping {
return true;
}
let step_size = 64 * 1024; let end = std::cmp::min(
self.sweep_cursor + step_size,
self.heap_base + self.heap_size,
);
self.sweep_cursor = end;
if self.sweep_cursor >= self.heap_base + self.heap_size {
self.sweeping = false;
true
} else {
false
}
}
}
#[derive(Debug, Clone)]
pub struct X86GCStackMapRecord {
pub id: u64,
pub instruction_offset: u64,
pub flags: u16,
pub locations: Vec<X86GCStackMapLocation>,
pub live_outs: Vec<X86GCStackMapLiveOut>,
pub function_index: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCStackMapLocation {
pub kind: X86GCStackMapLocationKind,
pub dwarf_reg: u16,
pub offset: i32,
pub size: u16,
pub reserved: u16,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCStackMapLocationKind {
Register = 1,
Direct = 2,
Indirect = 3,
Constant = 4,
ConstantIndex = 5,
}
impl X86GCStackMapLocationKind {
pub fn from_u8(value: u8) -> Option<Self> {
match value {
1 => Some(X86GCStackMapLocationKind::Register),
2 => Some(X86GCStackMapLocationKind::Direct),
3 => Some(X86GCStackMapLocationKind::Indirect),
4 => Some(X86GCStackMapLocationKind::Constant),
5 => Some(X86GCStackMapLocationKind::ConstantIndex),
_ => None,
}
}
pub fn to_u8(self) -> u8 {
self as u8
}
}
impl X86GCStackMapLocation {
pub fn new_register(dwarf_reg: u16, size: u16) -> Self {
X86GCStackMapLocation {
kind: X86GCStackMapLocationKind::Register,
dwarf_reg,
offset: 0,
size,
reserved: 0,
}
}
pub fn new_direct(frame_offset: i32, size: u16) -> Self {
X86GCStackMapLocation {
kind: X86GCStackMapLocationKind::Direct,
dwarf_reg: 0,
offset: frame_offset,
size,
reserved: 0,
}
}
pub fn new_indirect(dwarf_reg: u16, offset: i32, size: u16) -> Self {
X86GCStackMapLocation {
kind: X86GCStackMapLocationKind::Indirect,
dwarf_reg,
offset,
size,
reserved: 0,
}
}
pub fn new_constant(value: i32) -> Self {
X86GCStackMapLocation {
kind: X86GCStackMapLocationKind::Constant,
dwarf_reg: 0,
offset: value,
size: 8,
reserved: 0,
}
}
pub fn new_constant_index(index: i32, size: u16) -> Self {
X86GCStackMapLocation {
kind: X86GCStackMapLocationKind::ConstantIndex,
dwarf_reg: 0,
offset: index,
size,
reserved: 0,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCStackMapLiveOut {
pub dwarf_reg: u16,
pub reserved: u8,
pub size: u8,
}
impl X86GCStackMapLiveOut {
pub fn new(dwarf_reg: u16, size: u8) -> Self {
X86GCStackMapLiveOut {
dwarf_reg,
reserved: 0,
size,
}
}
}
#[derive(Debug, Clone)]
pub struct X86GCStackMap {
pub version: u8,
pub reserved1: u8,
pub reserved2: u16,
pub functions: Vec<X86GCStackMapFunction>,
pub constants: Vec<u64>,
pub records: Vec<X86GCStackMapRecord>,
pub func_to_records: HashMap<u32, Vec<usize>>,
}
#[derive(Debug, Clone)]
pub struct X86GCStackMapFunction {
pub function_address: u64,
pub stack_size: u64,
pub record_count: u64,
pub stack_sizes: Vec<X86GCStackSizeRecord>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCStackSizeRecord {
pub function_offset: u64,
pub stack_size: u64,
}
impl X86GCStackMapFunction {
pub fn new(function_address: u64, stack_size: u64) -> Self {
X86GCStackMapFunction {
function_address,
stack_size,
record_count: 0,
stack_sizes: Vec::new(),
}
}
pub fn add_stack_size(&mut self, offset: u64, size: u64) {
self.stack_sizes.push(X86GCStackSizeRecord {
function_offset: offset,
stack_size: size,
});
}
pub fn increment_records(&mut self) {
self.record_count += 1;
}
}
impl X86GCStackMap {
pub fn new() -> Self {
X86GCStackMap {
version: X86_GC_STACK_MAP_VERSION,
reserved1: 0,
reserved2: 0,
functions: Vec::new(),
constants: Vec::new(),
records: Vec::new(),
func_to_records: HashMap::new(),
}
}
pub fn add_function(&mut self, address: u64, stack_size: u64) -> u32 {
let idx = self.functions.len() as u32;
self.functions
.push(X86GCStackMapFunction::new(address, stack_size));
idx
}
pub fn add_constant(&mut self, value: u64) -> u32 {
let idx = self.constants.len() as u32;
self.constants.push(value);
idx
}
pub fn add_record(&mut self, record: X86GCStackMapRecord, function_index: u32) {
let record_index = self.records.len();
if let Some(func) = self.functions.get_mut(function_index as usize) {
func.increment_records();
}
self.func_to_records
.entry(function_index)
.or_insert_with(Vec::new)
.push(record_index);
self.records.push(record);
}
pub fn get_function_records(&self, function_index: u32) -> Option<&[usize]> {
self.func_to_records
.get(&function_index)
.map(|v| v.as_slice())
}
pub fn record_count(&self) -> usize {
self.records.len()
}
pub fn location_count(&self) -> usize {
self.records.iter().map(|r| r.locations.len()).sum()
}
pub fn emit_bytes(&self) -> Vec<u8> {
let mut buf = Vec::with_capacity(4096);
buf.push(self.version);
buf.push(self.reserved1);
buf.extend_from_slice(&self.reserved2.to_le_bytes());
buf.extend_from_slice(&(self.functions.len() as u32).to_le_bytes());
buf.extend_from_slice(&(self.constants.len() as u32).to_le_bytes());
buf.extend_from_slice(&(self.records.len() as u32).to_le_bytes());
for func in &self.functions {
buf.extend_from_slice(&func.function_address.to_le_bytes());
buf.extend_from_slice(&func.stack_size.to_le_bytes());
buf.extend_from_slice(&func.record_count.to_le_bytes());
}
for &c in &self.constants {
buf.extend_from_slice(&c.to_le_bytes());
}
for record in &self.records {
buf.extend_from_slice(&record.id.to_le_bytes());
buf.extend_from_slice(&record.instruction_offset.to_le_bytes());
buf.extend_from_slice(&record.flags.to_le_bytes());
buf.extend_from_slice(&(record.locations.len() as u16).to_le_bytes());
buf.extend_from_slice(&[0u8; 2]);
buf.extend_from_slice(&(record.live_outs.len() as u16).to_le_bytes());
for loc in &record.locations {
buf.push(loc.kind.to_u8());
buf.push(0); buf.extend_from_slice(&loc.size.to_le_bytes());
buf.extend_from_slice(&loc.dwarf_reg.to_le_bytes());
buf.extend_from_slice(&loc.reserved.to_le_bytes());
buf.extend_from_slice(&loc.offset.to_le_bytes());
}
let padding = (8 - (record.locations.len() * 3) % 8) % 8;
for _ in 0..padding {
buf.push(0);
}
for lo in &record.live_outs {
buf.extend_from_slice(&lo.dwarf_reg.to_le_bytes());
buf.push(lo.reserved);
buf.push(lo.size);
}
let lo_padding = (8 - (record.live_outs.len() * 4) % 8) % 8;
for _ in 0..lo_padding {
buf.push(0);
}
}
buf
}
pub fn emit_assembly(&self) -> String {
let mut s = String::new();
s.push_str(&format!("\t.section {},\n", X86_GC_STACK_MAP_SECTION));
s.push_str(&format!("\t.byte {}\t# version\n", self.version));
s.push_str(&format!("\t.byte {}\t# reserved\n", self.reserved1));
s.push_str(&format!("\t.short {}\t# reserved\n", self.reserved2));
s.push_str(&format!(
"\t.long {}\t# num_functions\n",
self.functions.len()
));
s.push_str(&format!(
"\t.long {}\t# num_constants\n",
self.constants.len()
));
s.push_str(&format!(
"\t.long {}\t# num_records\n\n",
self.records.len()
));
for (i, func) in self.functions.iter().enumerate() {
s.push_str(&format!("\t# Function {}\n", i));
s.push_str(&format!(
"\t.quad {}\t# function_address\n",
func.function_address
));
s.push_str(&format!("\t.quad {}\t# stack_size\n", func.stack_size));
s.push_str(&format!("\t.quad {}\t# record_count\n", func.record_count));
}
s
}
}
impl Default for X86GCStackMap {
fn default() -> Self {
X86GCStackMap::new()
}
}
#[derive(Debug, Clone)]
pub struct X86GCStackMapGenerator {
pub stack_map: X86GCStackMap,
pub current_function_index: Option<u32>,
pub current_function_name: Option<String>,
pub live_var_state: X86GCLiveVarState,
pub current_offset: u64,
pub current_frame_size: u64,
pub reg_names: HashMap<u16, String>,
pub next_id: u64,
}
#[derive(Debug, Clone)]
pub struct X86GCLiveVarState {
pub var_locations: HashMap<String, X86GCStackMapLocation>,
pub var_types: HashMap<String, String>,
pub gc_pointers: HashSet<String>,
pub stack_slots: BTreeMap<i32, u16>,
pub frame_size: u64,
}
impl X86GCLiveVarState {
pub fn new(frame_size: u64) -> Self {
X86GCLiveVarState {
var_locations: HashMap::new(),
var_types: HashMap::new(),
gc_pointers: HashSet::new(),
stack_slots: BTreeMap::new(),
frame_size,
}
}
pub fn record_register(&mut self, var_name: &str, dwarf_reg: u16, size: u16) {
self.var_locations.insert(
var_name.to_string(),
X86GCStackMapLocation::new_register(dwarf_reg, size),
);
}
pub fn record_stack(&mut self, var_name: &str, frame_offset: i32, size: u16) {
self.var_locations.insert(
var_name.to_string(),
X86GCStackMapLocation::new_direct(frame_offset, size),
);
self.stack_slots.insert(frame_offset, size);
}
pub fn mark_gc_pointer(&mut self, var_name: &str) {
self.gc_pointers.insert(var_name.to_string());
}
pub fn is_gc_pointer(&self, var_name: &str) -> bool {
self.gc_pointers.contains(var_name)
}
pub fn allocate_stack_slot(&mut self, var_name: &str, size: u16) -> i32 {
let mut offset = -(size as i32);
loop {
let mut conflicts = false;
for (&slot_offset, &slot_size) in &self.stack_slots {
let slot_end = slot_offset + slot_size as i32;
let new_end = offset + size as i32;
if !(new_end <= slot_offset || offset >= slot_end) {
conflicts = true;
break;
}
}
if !conflicts {
break;
}
offset -= 8; }
self.stack_slots.insert(offset, size);
self.var_locations.insert(
var_name.to_string(),
X86GCStackMapLocation::new_direct(offset, size),
);
offset
}
pub fn get_location(&self, var_name: &str) -> Option<&X86GCStackMapLocation> {
self.var_locations.get(var_name)
}
pub fn get_gc_pointer_locations(&self) -> Vec<X86GCStackMapLocation> {
self.gc_pointers
.iter()
.filter_map(|name| self.var_locations.get(name).copied())
.collect()
}
}
impl Default for X86GCLiveVarState {
fn default() -> Self {
X86GCLiveVarState::new(0)
}
}
impl X86GCStackMapGenerator {
pub fn new() -> Self {
X86GCStackMapGenerator {
stack_map: X86GCStackMap::new(),
current_function_index: None,
current_function_name: None,
live_var_state: X86GCLiveVarState::default(),
current_offset: 0,
current_frame_size: 0,
reg_names: HashMap::new(),
next_id: 1,
}
}
pub fn begin_function(&mut self, name: &str, address: u64) {
self.current_function_name = Some(name.to_string());
self.current_function_index = Some(
self.stack_map
.add_function(address, self.current_frame_size),
);
self.current_offset = 0;
self.live_var_state = X86GCLiveVarState::new(self.current_frame_size);
}
pub fn end_function(&mut self) {
self.current_function_name = None;
self.current_function_index = None;
}
pub fn advance_offset(&mut self, offset: u64) {
self.current_offset = offset;
}
pub fn generate_record(&mut self, is_call: bool) -> u64 {
let id = self.next_id;
self.next_id += 1;
let locations = self.live_var_state.get_gc_pointer_locations();
let record = X86GCStackMapRecord {
id,
instruction_offset: self.current_offset,
flags: if is_call { 1 } else { 0 },
locations,
live_outs: Vec::new(),
function_index: self.current_function_index.unwrap_or(0),
};
if let Some(func_idx) = self.current_function_index {
self.stack_map.add_record(record, func_idx);
}
id
}
pub fn generate_for_patchpoint(&mut self, pp_id: u64, target: &str) -> u64 {
let id = self.next_id;
self.next_id += 1;
let locations = self.live_var_state.get_gc_pointer_locations();
let record = X86GCStackMapRecord {
id,
instruction_offset: self.current_offset,
flags: 2, locations,
live_outs: Vec::new(),
function_index: self.current_function_index.unwrap_or(0),
};
let _ = (pp_id, target);
if let Some(func_idx) = self.current_function_index {
self.stack_map.add_record(record, func_idx);
}
id
}
pub fn generate_for_statepoint(&mut self, sp_id: u64, callee: &str, gc_args: &[&str]) -> u64 {
let id = self.next_id;
self.next_id += 1;
let mut locations = self.live_var_state.get_gc_pointer_locations();
for &arg_name in gc_args {
if let Some(&loc) = self.live_var_state.get_location(arg_name) {
locations.push(loc);
}
}
let record = X86GCStackMapRecord {
id,
instruction_offset: self.current_offset,
flags: 4, locations,
live_outs: Vec::new(),
function_index: self.current_function_index.unwrap_or(0),
};
let _ = (sp_id, callee);
if let Some(func_idx) = self.current_function_index {
self.stack_map.add_record(record, func_idx);
}
id
}
pub fn finalize(&self) -> X86GCStackMap {
self.stack_map.clone()
}
pub fn frame_size(&self) -> u64 {
self.current_frame_size
}
pub fn set_frame_size(&mut self, size: u64) {
self.current_frame_size = size;
}
}
impl Default for X86GCStackMapGenerator {
fn default() -> Self {
X86GCStackMapGenerator::new()
}
}
#[derive(Debug, Clone)]
pub struct X86GCStackMapParser {
pub stack_map: X86GCStackMap,
pub id_index: HashMap<u64, usize>,
pub parsed: bool,
pub warnings: Vec<String>,
}
impl X86GCStackMapParser {
pub fn new() -> Self {
X86GCStackMapParser {
stack_map: X86GCStackMap::new(),
id_index: HashMap::new(),
parsed: false,
warnings: Vec::new(),
}
}
pub fn parse(&mut self, data: &[u8]) -> bool {
if data.len() < 16 {
self.warnings
.push("Data too short for stack map header".to_string());
return false;
}
let mut offset = 0;
let version = data[offset];
offset += 1;
let reserved1 = data[offset];
offset += 1;
let reserved2 = u16::from_le_bytes([data[offset], data[offset + 1]]);
offset += 2;
let num_functions = u32::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
]) as usize;
offset += 4;
let num_constants = u32::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
]) as usize;
offset += 4;
let num_records = u32::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
]) as usize;
offset += 4;
self.stack_map.version = version;
self.stack_map.reserved1 = reserved1;
self.stack_map.reserved2 = reserved2;
for _ in 0..num_functions {
if offset + 24 > data.len() {
self.warnings.push("Truncated function record".to_string());
return false;
}
let addr = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
offset += 8;
let stack_size = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
offset += 8;
let record_count = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
offset += 8;
let mut func = X86GCStackMapFunction::new(addr, stack_size);
func.record_count = record_count;
self.stack_map.functions.push(func);
}
for _ in 0..num_constants {
if offset + 8 > data.len() {
self.warnings.push("Truncated constant entry".to_string());
return false;
}
let val = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
offset += 8;
self.stack_map.constants.push(val);
}
for _ in 0..num_records {
if offset + 16 > data.len() {
break;
}
let id = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
offset += 8;
let inst_offset = u64::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
data[offset + 4],
data[offset + 5],
data[offset + 6],
data[offset + 7],
]);
offset += 8;
let flags = u16::from_le_bytes([data[offset], data[offset + 1]]);
offset += 2;
let num_locs = u16::from_le_bytes([data[offset], data[offset + 1]]) as usize;
offset += 2;
offset += 4;
let num_live_outs = u16::from_le_bytes([data[offset], data[offset + 1]]) as usize;
offset += 2;
offset += 2;
let mut locations = Vec::with_capacity(num_locs);
for _ in 0..num_locs {
if offset + 12 > data.len() {
break;
}
let kind = X86GCStackMapLocationKind::from_u8(data[offset])
.unwrap_or(X86GCStackMapLocationKind::Register);
offset += 1;
let _reserved_loc = data[offset];
offset += 1;
let size = u16::from_le_bytes([data[offset], data[offset + 1]]);
offset += 2;
let dwarf_reg = u16::from_le_bytes([data[offset], data[offset + 1]]);
offset += 2;
let reserved = u16::from_le_bytes([data[offset], data[offset + 1]]);
offset += 2;
let loc_offset = i32::from_le_bytes([
data[offset],
data[offset + 1],
data[offset + 2],
data[offset + 3],
]);
offset += 4;
locations.push(X86GCStackMapLocation {
kind,
dwarf_reg,
offset: loc_offset,
size,
reserved,
});
}
let mut live_outs = Vec::with_capacity(num_live_outs);
for _ in 0..num_live_outs {
if offset + 4 > data.len() {
break;
}
let dwarf_reg = u16::from_le_bytes([data[offset], data[offset + 1]]);
offset += 2;
let reserved_lo = data[offset];
offset += 1;
let size_lo = data[offset];
offset += 1;
live_outs.push(X86GCStackMapLiveOut {
dwarf_reg,
reserved: reserved_lo,
size: size_lo,
});
}
let record_idx = self.stack_map.records.len();
self.id_index.insert(id, record_idx);
self.stack_map.records.push(X86GCStackMapRecord {
id,
instruction_offset: inst_offset,
flags,
locations,
live_outs,
function_index: 0,
});
}
self.parsed = true;
true
}
pub fn lookup_by_id(&self, id: u64) -> Option<&X86GCStackMapRecord> {
self.id_index
.get(&id)
.and_then(|&idx| self.stack_map.records.get(idx))
}
pub fn all_ids(&self) -> Vec<u64> {
self.id_index.keys().copied().collect()
}
pub fn record_count(&self) -> usize {
self.stack_map.records.len()
}
}
impl Default for X86GCStackMapParser {
fn default() -> Self {
X86GCStackMapParser::new()
}
}
#[derive(Debug, Clone)]
pub struct X86GCSafepoints {
pub enabled: bool,
pub polling_page_address: u64,
pub polling_page_size: u64,
pub safepoint_count: u64,
pub insert_at_entry: bool,
pub insert_at_backedge: bool,
pub insert_after_calls: bool,
pub backedge_frequency: u32,
pub stats: X86GCSafepointStats,
pub safepoints: HashMap<String, Vec<X86GCSafepointLocation>>,
}
#[derive(Debug, Clone, Default)]
pub struct X86GCSafepointStats {
pub entry_safepoints: u64,
pub backedge_safepoints: u64,
pub call_safepoints: u64,
pub statepoint_safepoints: u64,
pub coop_suspend_points: u64,
pub exit_safepoints: u64,
}
impl X86GCSafepointStats {
pub fn total(&self) -> u64 {
self.entry_safepoints
+ self.backedge_safepoints
+ self.call_safepoints
+ self.statepoint_safepoints
+ self.coop_suspend_points
+ self.exit_safepoints
}
}
#[derive(Debug, Clone)]
pub struct X86GCSafepointLocation {
pub kind: X86GCSafepointKind,
pub function_name: String,
pub instruction_offset: u64,
pub stack_map_id: Option<u64>,
pub machine_instr: Option<MachineInstr>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCSafepointKind {
FunctionEntry,
LoopBackedge,
AfterCall,
Statepoint,
CooperativeSuspend,
FunctionExit,
}
impl fmt::Display for X86GCSafepointKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCSafepointKind::FunctionEntry => write!(f, "entry"),
X86GCSafepointKind::LoopBackedge => write!(f, "backedge"),
X86GCSafepointKind::AfterCall => write!(f, "after-call"),
X86GCSafepointKind::Statepoint => write!(f, "statepoint"),
X86GCSafepointKind::CooperativeSuspend => write!(f, "coop-suspend"),
X86GCSafepointKind::FunctionExit => write!(f, "exit"),
}
}
}
impl X86GCSafepoints {
pub fn new(polling_page_address: u64) -> Self {
X86GCSafepoints {
enabled: true,
polling_page_address,
polling_page_size: X86_GC_POLLING_PAGE_SIZE,
safepoint_count: 0,
insert_at_entry: true,
insert_at_backedge: true,
insert_after_calls: false,
backedge_frequency: 1,
stats: X86GCSafepointStats::default(),
safepoints: HashMap::new(),
}
}
pub fn set_enabled(&mut self, enabled: bool) {
self.enabled = enabled;
}
pub fn set_polling_page(&mut self, address: u64) {
self.polling_page_address = address;
}
pub fn configure(
&mut self,
insert_at_entry: bool,
insert_at_backedge: bool,
insert_after_calls: bool,
backedge_frequency: u32,
) {
self.insert_at_entry = insert_at_entry;
self.insert_at_backedge = insert_at_backedge;
self.insert_after_calls = insert_after_calls;
self.backedge_frequency = backedge_frequency;
}
pub fn generate_poll_bytes(&self) -> Vec<u8> {
let mut code = Vec::with_capacity(8);
code.push(0x48); code.push(0xA1); code.extend_from_slice(&self.polling_page_address.to_le_bytes());
code
}
pub fn arm_polling_page(&self) {
}
pub fn disarm_polling_page(&self) {
}
pub fn insert_entry_safepoint(
&mut self,
function_name: &str,
generator: &mut X86GCStackMapGenerator,
) -> Option<u64> {
if !self.enabled || !self.insert_at_entry {
return None;
}
self.stats.entry_safepoints += 1;
self.safepoint_count += 1;
let record_id = generator.generate_record(false);
self.safepoints
.entry(function_name.to_string())
.or_insert_with(Vec::new)
.push(X86GCSafepointLocation {
kind: X86GCSafepointKind::FunctionEntry,
function_name: function_name.to_string(),
instruction_offset: 0,
stack_map_id: Some(record_id),
machine_instr: None,
});
Some(record_id)
}
pub fn insert_backedge_safepoint(
&mut self,
function_name: &str,
offset: u64,
generator: &mut X86GCStackMapGenerator,
) -> Option<u64> {
if !self.enabled || !self.insert_at_backedge {
return None;
}
self.stats.backedge_safepoints += 1;
self.safepoint_count += 1;
generator.advance_offset(offset);
let record_id = generator.generate_record(false);
self.safepoints
.entry(function_name.to_string())
.or_insert_with(Vec::new)
.push(X86GCSafepointLocation {
kind: X86GCSafepointKind::LoopBackedge,
function_name: function_name.to_string(),
instruction_offset: offset,
stack_map_id: Some(record_id),
machine_instr: None,
});
Some(record_id)
}
pub fn insert_call_safepoint(
&mut self,
function_name: &str,
offset: u64,
generator: &mut X86GCStackMapGenerator,
) -> Option<u64> {
if !self.enabled || !self.insert_after_calls {
return None;
}
self.stats.call_safepoints += 1;
self.safepoint_count += 1;
generator.advance_offset(offset);
let record_id = generator.generate_record(true);
self.safepoints
.entry(function_name.to_string())
.or_insert_with(Vec::new)
.push(X86GCSafepointLocation {
kind: X86GCSafepointKind::AfterCall,
function_name: function_name.to_string(),
instruction_offset: offset,
stack_map_id: Some(record_id),
machine_instr: None,
});
Some(record_id)
}
pub fn record_statepoint(&mut self, function_name: &str, offset: u64, stack_map_id: u64) {
self.stats.statepoint_safepoints += 1;
self.safepoint_count += 1;
self.safepoints
.entry(function_name.to_string())
.or_insert_with(Vec::new)
.push(X86GCSafepointLocation {
kind: X86GCSafepointKind::Statepoint,
function_name: function_name.to_string(),
instruction_offset: offset,
stack_map_id: Some(stack_map_id),
machine_instr: None,
});
}
pub fn get_stats(&self) -> &X86GCSafepointStats {
&self.stats
}
pub fn get_function_safepoints(
&self,
function_name: &str,
) -> Option<&[X86GCSafepointLocation]> {
self.safepoints.get(function_name).map(|v| v.as_slice())
}
pub fn should_insert_backedge(&self, backedge_count: u32) -> bool {
self.insert_at_backedge && (backedge_count % self.backedge_frequency == 0)
}
pub fn reset(&mut self) {
self.safepoint_count = 0;
self.stats = X86GCSafepointStats::default();
self.safepoints.clear();
}
}
impl Default for X86GCSafepoints {
fn default() -> Self {
X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86GCReferenceType {
Strong,
Weak,
Soft,
Phantom,
Ephemeron,
Finalizer,
}
impl fmt::Display for X86GCReferenceType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCReferenceType::Strong => write!(f, "strong"),
X86GCReferenceType::Weak => write!(f, "weak"),
X86GCReferenceType::Soft => write!(f, "soft"),
X86GCReferenceType::Phantom => write!(f, "phantom"),
X86GCReferenceType::Ephemeron => write!(f, "ephemeron"),
X86GCReferenceType::Finalizer => write!(f, "finalizer"),
}
}
}
#[derive(Debug, Clone)]
pub struct X86GCFinalizerEntry {
pub object_address: u64,
pub type_name: Option<String>,
pub finalizer_address: u64,
pub registered: bool,
pub finalized: bool,
pub priority: u32,
pub generation: u64,
}
#[derive(Debug, Clone)]
pub struct X86GCWeakReference {
pub reference_address: u64,
pub referent_address: u64,
pub cleared: bool,
pub notify_on_clear: bool,
}
#[derive(Debug, Clone)]
pub struct X86GCPhantomReference {
pub reference_address: u64,
pub referent_address: u64,
pub cleared: bool,
pub finalized: bool,
}
#[derive(Debug, Clone)]
pub struct X86GCEphemeron {
pub key_address: u64,
pub value_address: u64,
pub key_reachable: bool,
pub value_cleared: bool,
}
#[derive(Debug, Clone)]
pub struct X86GCFinalization {
pub finalizer_queue: VecDeque<X86GCFinalizerEntry>,
pub finalized_objects: HashSet<u64>,
pub weak_references: HashMap<u64, X86GCWeakReference>,
pub phantom_references: HashMap<u64, X86GCPhantomReference>,
pub ephemerons: Vec<X86GCEphemeron>,
pub enabled: bool,
pub total_finalizers_run: u64,
pub total_weak_cleared: u64,
pub total_phantom_processed: u64,
pub total_ephemerons_cleared: u64,
}
impl X86GCFinalization {
pub fn new() -> Self {
X86GCFinalization {
finalizer_queue: VecDeque::new(),
finalized_objects: HashSet::new(),
weak_references: HashMap::new(),
phantom_references: HashMap::new(),
ephemerons: Vec::new(),
enabled: true,
total_finalizers_run: 0,
total_weak_cleared: 0,
total_phantom_processed: 0,
total_ephemerons_cleared: 0,
}
}
pub fn register_finalizer(
&mut self,
object_address: u64,
finalizer_address: u64,
priority: u32,
) {
self.finalizer_queue.push_back(X86GCFinalizerEntry {
object_address,
type_name: None,
finalizer_address,
registered: true,
finalized: false,
priority,
generation: 0,
});
}
pub fn process_finalizers(&mut self, max_per_cycle: usize) -> usize {
if !self.enabled {
return 0;
}
let mut processed = 0;
let mut remaining = VecDeque::new();
while let Some(entry) = self.finalizer_queue.pop_front() {
if processed >= max_per_cycle {
remaining.push_back(entry);
continue;
}
if self.finalized_objects.contains(&entry.object_address) {
continue; }
self.finalized_objects.insert(entry.object_address);
self.total_finalizers_run += 1;
processed += 1;
}
self.finalizer_queue.append(&mut remaining);
processed
}
pub fn has_finalizer(&self, object_address: u64) -> bool {
self.finalizer_queue
.iter()
.any(|e| e.object_address == object_address && !e.finalized)
}
pub fn pending_finalizer_count(&self) -> usize {
self.finalizer_queue.len()
}
pub fn register_weak_reference(
&mut self,
reference_address: u64,
referent_address: u64,
notify: bool,
) {
self.weak_references.insert(
reference_address,
X86GCWeakReference {
reference_address,
referent_address,
cleared: false,
notify_on_clear: notify,
},
);
}
pub fn process_weak_references(&mut self, is_marked: &dyn Fn(u64) -> bool) -> usize {
if !self.enabled {
return 0;
}
let mut cleared = 0;
for weak_ref in self.weak_references.values_mut() {
if !weak_ref.cleared && !is_marked(weak_ref.referent_address) {
weak_ref.cleared = true;
cleared += 1;
}
}
self.total_weak_cleared += cleared as u64;
cleared
}
pub fn live_weak_references(&self, is_marked: &dyn Fn(u64) -> bool) -> Vec<u64> {
self.weak_references
.values()
.filter(|wr| !wr.cleared && is_marked(wr.referent_address))
.map(|wr| wr.reference_address)
.collect()
}
pub fn register_phantom_reference(&mut self, reference_address: u64, referent_address: u64) {
self.phantom_references.insert(
reference_address,
X86GCPhantomReference {
reference_address,
referent_address,
cleared: false,
finalized: false,
},
);
}
pub fn process_phantom_references(&mut self, is_marked: &dyn Fn(u64) -> bool) -> usize {
if !self.enabled {
return 0;
}
let mut processed = 0;
for phantom in self.phantom_references.values_mut() {
if !phantom.cleared
&& !is_marked(phantom.referent_address)
&& self.finalized_objects.contains(&phantom.referent_address)
{
phantom.cleared = true;
phantom.finalized = true;
processed += 1;
}
}
self.total_phantom_processed += processed as u64;
processed
}
pub fn register_ephemeron(&mut self, key_address: u64, value_address: u64) {
self.ephemerons.push(X86GCEphemeron {
key_address,
value_address,
key_reachable: false,
value_cleared: false,
});
}
pub fn process_ephemerons(&mut self, is_marked: &dyn Fn(u64) -> bool) -> usize {
if !self.enabled {
return 0;
}
let mut cleared = 0;
for eph in &mut self.ephemerons {
eph.key_reachable = is_marked(eph.key_address);
}
for eph in &mut self.ephemerons {
if !eph.key_reachable && !eph.value_cleared {
eph.value_cleared = true;
cleared += 1;
}
}
for eph in &self.ephemerons {
if eph.key_reachable && !is_marked(eph.value_address) {
}
}
self.total_ephemerons_cleared += cleared as u64;
cleared
}
pub fn cleanup(&mut self) {
self.weak_references.retain(|_, wr| !wr.cleared);
self.phantom_references.retain(|_, pr| !pr.finalized);
self.ephemerons.retain(|e| !e.value_cleared);
}
pub fn reset(&mut self) {
self.finalizer_queue.clear();
self.finalized_objects.clear();
self.weak_references.clear();
self.phantom_references.clear();
self.ephemerons.clear();
self.total_finalizers_run = 0;
self.total_weak_cleared = 0;
self.total_phantom_processed = 0;
self.total_ephemerons_cleared = 0;
}
}
impl Default for X86GCFinalization {
fn default() -> Self {
X86GCFinalization::new()
}
}
#[derive(Debug, Clone)]
pub struct X86GCStackWalker {
pub stack_pointer: u64,
pub frame_pointer: u64,
pub instruction_pointer: u64,
pub stack_bottom: u64,
pub stack_top: u64,
pub use_frame_pointer: bool,
pub fallback_conservative: bool,
pub frames: Vec<X86GCStackFrame>,
pub discovered_roots: Vec<X86GCRoot>,
pub walk_complete: bool,
pub frame_count: u64,
}
#[derive(Debug, Clone)]
pub struct X86GCStackFrame {
pub index: u64,
pub return_address: u64,
pub frame_pointer: u64,
pub stack_pointer: u64,
pub frame_size: u64,
pub function_name: Option<String>,
pub has_stack_map: bool,
pub frame_roots: Vec<X86GCRoot>,
pub is_leaf: bool,
}
impl X86GCStackWalker {
pub fn new(rsp: u64, rbp: u64, rip: u64, stack_bottom: u64, stack_top: u64) -> Self {
X86GCStackWalker {
stack_pointer: rsp,
frame_pointer: rbp,
instruction_pointer: rip,
stack_bottom,
stack_top,
use_frame_pointer: true,
fallback_conservative: true,
frames: Vec::new(),
discovered_roots: Vec::new(),
walk_complete: false,
frame_count: 0,
}
}
pub fn walk_frames(&mut self) {
let mut current_rbp = self.frame_pointer;
let mut frame_idx = 0u64;
loop {
if current_rbp < self.stack_bottom || current_rbp >= self.stack_top {
break;
}
let frame = X86GCStackFrame {
index: frame_idx,
return_address: current_rbp.wrapping_add(8), frame_pointer: current_rbp,
stack_pointer: current_rbp,
frame_size: 0, function_name: None,
has_stack_map: false,
frame_roots: Vec::new(),
is_leaf: frame_idx == 0,
};
self.frames.push(frame);
frame_idx += 1;
if frame_idx > 100 {
break;
}
current_rbp = current_rbp.wrapping_add(0x40);
if current_rbp == 0 {
break;
}
}
self.frame_count = frame_idx;
self.walk_complete = true;
}
pub fn scan_frame_precise(
&mut self,
frame_idx: usize,
stack_map: &X86GCStackMap,
function_index: u32,
) -> Vec<X86GCRoot> {
let mut roots = Vec::new();
if let Some(record_indices) = stack_map.get_function_records(function_index) {
for &rec_idx in record_indices {
if let Some(record) = stack_map.records.get(rec_idx) {
for loc in &record.locations {
let root = match loc.kind {
X86GCStackMapLocationKind::Register => {
X86GCRoot::new_register((roots.len() + 1) as u32, loc.dwarf_reg)
}
X86GCStackMapLocationKind::Direct => {
X86GCRoot::new_base((roots.len() + 1) as u32, loc.offset)
}
_ => continue,
};
roots.push(root);
}
}
}
}
if frame_idx < self.frames.len() {
self.frames[frame_idx].has_stack_map = true;
self.frames[frame_idx].frame_roots = roots.clone();
}
roots
}
pub fn scan_frame_conservative(
&mut self,
frame_idx: usize,
heap_start: u64,
heap_end: u64,
) -> Vec<X86GCRootCandidate> {
let mut candidates = Vec::new();
if frame_idx >= self.frames.len() {
return candidates;
}
let frame = &self.frames[frame_idx];
let frame_start = frame.stack_pointer;
let frame_end = frame.frame_pointer;
let mut addr = frame_start & !7u64; let mut root_id = 1u32;
while addr < frame_end && addr < self.stack_top {
candidates.push(X86GCRootCandidate {
raw_value: 0, stack_address: addr,
frame_offset: (addr as i64 - frame.frame_pointer as i64) as i32,
is_valid_heap_ptr: false, scan_mode: X86GCScanMode::Conservative,
});
root_id += 1;
addr += 8;
}
candidates
}
pub fn collect_all_roots(
&mut self,
stack_map: Option<&X86GCStackMap>,
heap_start: u64,
heap_end: u64,
) {
self.discovered_roots.clear();
for frame_idx in 0..self.frames.len() {
if let Some(sm) = stack_map {
let precise_roots = self.scan_frame_precise(frame_idx, sm, 0);
if !precise_roots.is_empty() {
self.discovered_roots.extend(precise_roots);
continue;
}
}
if self.fallback_conservative {
let candidates = self.scan_frame_conservative(frame_idx, heap_start, heap_end);
let _ = candidates;
}
}
}
pub fn total_frames(&self) -> u64 {
self.frame_count
}
pub fn frame_roots(&self, frame_idx: usize) -> Option<&[X86GCRoot]> {
self.frames.get(frame_idx).map(|f| f.frame_roots.as_slice())
}
}
impl Default for X86GCStackWalker {
fn default() -> Self {
X86GCStackWalker::new(0, 0, 0, 0, 0)
}
}
#[derive(Debug, Clone)]
pub struct X86GCMemoryPressure {
pub heap_occupancy: u64,
pub max_heap_size: u64,
pub gc_threshold: f64,
pub pressure_level: X86GCPressureLevel,
pub allocated_since_gc: u64,
pub freed_since_gc: u64,
pub stats: X86GCMemoryStats,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum X86GCPressureLevel {
Low = 0,
Normal = 1,
High = 2,
Critical = 3,
}
#[derive(Debug, Clone, Default)]
pub struct X86GCMemoryStats {
pub total_collections: u64,
pub alloc_failure_collections: u64,
pub threshold_collections: u64,
pub gc_time_units: u64,
pub total_bytes_promoted: u64,
pub total_bytes_tenured: u64,
pub avg_pause_time: f64,
pub max_pause_time: u64,
pub min_pause_time: u64,
pub fragmentation_ratio: f64,
}
impl X86GCMemoryPressure {
pub fn new(max_heap_size: u64) -> Self {
X86GCMemoryPressure {
heap_occupancy: 0,
max_heap_size,
gc_threshold: X86_GC_MIN_HEAP_OCCUPANCY,
pressure_level: X86GCPressureLevel::Low,
allocated_since_gc: 0,
freed_since_gc: 0,
stats: X86GCMemoryStats::default(),
}
}
pub fn record_allocation(&mut self, size: u64) {
self.heap_occupancy += size;
self.allocated_since_gc += size;
self.update_pressure();
}
pub fn record_free(&mut self, size: u64) {
self.heap_occupancy = self.heap_occupancy.saturating_sub(size);
self.freed_since_gc += size;
}
fn update_pressure(&mut self) {
let ratio = if self.max_heap_size > 0 {
self.heap_occupancy as f64 / self.max_heap_size as f64
} else {
0.0
};
self.pressure_level = if ratio >= 0.95 {
X86GCPressureLevel::Critical
} else if ratio >= self.gc_threshold {
X86GCPressureLevel::High
} else if ratio >= self.gc_threshold * 0.5 {
X86GCPressureLevel::Normal
} else {
X86GCPressureLevel::Low
};
}
pub fn should_collect(&self) -> bool {
self.pressure_level >= X86GCPressureLevel::High
}
pub fn needs_immediate_gc(&self) -> bool {
self.pressure_level >= X86GCPressureLevel::Critical
}
pub fn after_gc(&mut self, freed_bytes: u64) {
self.freed_since_gc = 0;
self.heap_occupancy = self.heap_occupancy.saturating_sub(freed_bytes);
self.update_pressure();
self.stats.total_collections += 1;
}
pub fn occupancy_ratio(&self) -> f64 {
if self.max_heap_size > 0 {
self.heap_occupancy as f64 / self.max_heap_size as f64
} else {
0.0
}
}
pub fn available_bytes(&self) -> u64 {
self.max_heap_size.saturating_sub(self.heap_occupancy)
}
}
impl Default for X86GCMemoryPressure {
fn default() -> Self {
X86GCMemoryPressure::new(1024 * 1024 * 1024)
}
}
#[derive(Debug, Clone)]
pub struct X86GCThreadSuspension {
pub suspended: Arc<AtomicBool>,
pub threads_at_safepoint: Arc<AtomicU64>,
pub total_threads: Arc<AtomicU64>,
pub state: X86GCSuspensionState,
pub cooperative: bool,
pub signal_preemption: bool,
pub suspension_timeout_us: u64,
pub preempted_threads: HashSet<u32>,
pub thread_states: HashMap<u32, X86GCThreadState>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCSuspensionState {
Running,
SuspensionRequested,
WaitingForThreads,
Suspended,
Resuming,
}
#[derive(Debug, Clone)]
pub struct X86GCThreadState {
pub thread_id: u32,
pub at_safepoint: bool,
pub preempted: bool,
pub saved_rsp: u64,
pub saved_rbp: u64,
pub saved_rip: u64,
pub has_pending_roots: bool,
}
impl X86GCThreadSuspension {
pub fn new(cooperative: bool, signal_preemption: bool) -> Self {
X86GCThreadSuspension {
suspended: Arc::new(AtomicBool::new(false)),
threads_at_safepoint: Arc::new(AtomicU64::new(0)),
total_threads: Arc::new(AtomicU64::new(0)),
state: X86GCSuspensionState::Running,
cooperative,
signal_preemption,
suspension_timeout_us: 100_000, preempted_threads: HashSet::new(),
thread_states: HashMap::new(),
}
}
pub fn register_thread(&mut self, thread_id: u32) {
self.thread_states.insert(
thread_id,
X86GCThreadState {
thread_id,
at_safepoint: false,
preempted: false,
saved_rsp: 0,
saved_rbp: 0,
saved_rip: 0,
has_pending_roots: false,
},
);
self.total_threads.fetch_add(1, Ordering::SeqCst);
}
pub fn thread_reached_safepoint(&mut self, thread_id: u32) {
if let Some(state) = self.thread_states.get_mut(&thread_id) {
if !state.at_safepoint {
state.at_safepoint = true;
self.threads_at_safepoint.fetch_add(1, Ordering::SeqCst);
}
}
}
pub fn thread_left_safepoint(&mut self, thread_id: u32) {
if let Some(state) = self.thread_states.get_mut(&thread_id) {
if state.at_safepoint {
state.at_safepoint = false;
self.threads_at_safepoint.fetch_sub(1, Ordering::SeqCst);
}
}
}
pub fn request_suspension(&mut self) {
self.state = X86GCSuspensionState::SuspensionRequested;
self.suspended.store(true, Ordering::SeqCst);
}
pub fn all_threads_at_safepoints(&self) -> bool {
let at_sp = self.threads_at_safepoint.load(Ordering::SeqCst);
let total = self.total_threads.load(Ordering::SeqCst);
at_sp >= total
}
pub fn enter_suspended(&mut self) {
self.state = X86GCSuspensionState::Suspended;
}
pub fn resume_all(&mut self) {
self.state = X86GCSuspensionState::Resuming;
self.suspended.store(false, Ordering::SeqCst);
for state in self.thread_states.values_mut() {
state.at_safepoint = false;
}
self.threads_at_safepoint.store(0, Ordering::SeqCst);
self.state = X86GCSuspensionState::Running;
}
pub fn should_yield(&self) -> bool {
self.suspended.load(Ordering::SeqCst)
}
pub fn preempt_thread(&mut self, thread_id: u32) {
if self.signal_preemption {
self.preempted_threads.insert(thread_id);
if let Some(state) = self.thread_states.get_mut(&thread_id) {
state.preempted = true;
}
}
}
pub fn running_threads(&self) -> u64 {
let total = self.total_threads.load(Ordering::SeqCst);
let at_sp = self.threads_at_safepoint.load(Ordering::SeqCst);
total.saturating_sub(at_sp)
}
}
impl Default for X86GCThreadSuspension {
fn default() -> Self {
X86GCThreadSuspension::new(true, false)
}
}
#[derive(Debug, Clone)]
pub struct X86GCDeoptimization {
pub reason: X86GCDeoptReason,
pub bytecode_offset: u32,
pub method_name: String,
pub registers: HashMap<u16, u64>,
pub stack_slots: Vec<X86GCDeoptStackSlot>,
pub local_values: Vec<X86GCDeoptLocal>,
pub inline_depth: u32,
pub is_reexecute: bool,
pub bundle_size: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCDeoptReason {
None,
SpeculativeGuard,
TypeCheck,
BoundsCheck,
NullCheck,
DivZero,
ClassCheck,
VirtualCall,
InlineCacheMiss,
Explicit,
Debug,
}
impl X86GCDeoptReason {
pub fn description(&self) -> &str {
match self {
X86GCDeoptReason::None => "none",
X86GCDeoptReason::SpeculativeGuard => "speculative guard failed",
X86GCDeoptReason::TypeCheck => "type check failed",
X86GCDeoptReason::BoundsCheck => "bounds check failed",
X86GCDeoptReason::NullCheck => "null check failed",
X86GCDeoptReason::DivZero => "division by zero",
X86GCDeoptReason::ClassCheck => "class check failed",
X86GCDeoptReason::VirtualCall => "virtual call guard failed",
X86GCDeoptReason::InlineCacheMiss => "inline cache miss",
X86GCDeoptReason::Explicit => "explicit deoptimization",
X86GCDeoptReason::Debug => "debug deoptimization",
}
}
}
#[derive(Debug, Clone)]
pub struct X86GCDeoptStackSlot {
pub offset: i32,
pub value: u64,
pub is_reference: bool,
pub size: u8,
}
#[derive(Debug, Clone)]
pub struct X86GCDeoptLocal {
pub name: String,
pub type_name: String,
pub value: u64,
pub is_reference: bool,
}
impl X86GCDeoptimization {
pub fn new(reason: X86GCDeoptReason, method_name: &str, bytecode_offset: u32) -> Self {
X86GCDeoptimization {
reason,
bytecode_offset,
method_name: method_name.to_string(),
registers: HashMap::new(),
stack_slots: Vec::new(),
local_values: Vec::new(),
inline_depth: 0,
is_reexecute: false,
bundle_size: 0,
}
}
pub fn record_register(&mut self, dwarf_reg: u16, value: u64) {
self.registers.insert(dwarf_reg, value);
}
pub fn record_stack_slot(&mut self, offset: i32, value: u64, size: u8, is_ref: bool) {
self.stack_slots.push(X86GCDeoptStackSlot {
offset,
value,
is_reference: is_ref,
size,
});
}
pub fn record_local(&mut self, name: &str, type_name: &str, value: u64, is_ref: bool) {
self.local_values.push(X86GCDeoptLocal {
name: name.to_string(),
type_name: type_name.to_string(),
value,
is_reference: is_ref,
});
}
pub fn mark_reexecute(&mut self) {
self.is_reexecute = true;
}
pub fn with_inline_depth(mut self, depth: u32) -> Self {
self.inline_depth = depth;
self
}
pub fn gc_references(&self) -> Vec<u64> {
let mut refs = Vec::new();
for (_, &value) in self.registers.iter() {
if value != 0 {
refs.push(value);
}
}
for slot in &self.stack_slots {
if slot.is_reference && slot.value != 0 {
refs.push(slot.value);
}
}
for local in &self.local_values {
if local.is_reference && local.value != 0 {
refs.push(local.value);
}
}
refs
}
}
#[derive(Debug, Clone, Default)]
pub struct X86GCStatistics {
pub total_gc_cycles: u64,
pub minor_collections: u64,
pub major_collections: u64,
pub full_collections: u64,
pub concurrent_marking_cycles: u64,
pub total_bytes_allocated: u64,
pub total_bytes_freed: u64,
pub total_bytes_copied: u64,
pub total_bytes_promoted: u64,
pub total_finalizers_executed: u64,
pub total_weak_refs_cleared: u64,
pub total_phantom_processed: u64,
pub total_write_barriers: u64,
pub total_read_barriers: u64,
pub total_satb_logged: u64,
pub total_tlab_refills: u64,
pub total_slow_path_allocs: u64,
pub avg_gc_pause_time: f64,
pub max_gc_pause_time: u64,
pub total_gc_time: u64,
pub total_safepoints_hit: u64,
pub total_poll_page_faults: u64,
pub last_fragmentation_ratio: f64,
pub heap_occupancy_ratio: f64,
pub oom_events: u64,
pub collection_efficiency: f64,
}
impl X86GCStatistics {
pub fn new() -> Self {
X86GCStatistics {
total_gc_cycles: 0,
minor_collections: 0,
major_collections: 0,
full_collections: 0,
concurrent_marking_cycles: 0,
total_bytes_allocated: 0,
total_bytes_freed: 0,
total_bytes_copied: 0,
total_bytes_promoted: 0,
total_finalizers_executed: 0,
total_weak_refs_cleared: 0,
total_phantom_processed: 0,
total_write_barriers: 0,
total_read_barriers: 0,
total_satb_logged: 0,
total_tlab_refills: 0,
total_slow_path_allocs: 0,
avg_gc_pause_time: 0.0,
max_gc_pause_time: 0,
total_gc_time: 0,
total_safepoints_hit: 0,
total_poll_page_faults: 0,
last_fragmentation_ratio: 0.0,
heap_occupancy_ratio: 0.0,
oom_events: 0,
collection_efficiency: 0.0,
}
}
pub fn record_gc_cycle(
&mut self,
is_minor: bool,
is_major: bool,
pause_time: u64,
bytes_freed: u64,
) {
self.total_gc_cycles += 1;
if is_minor {
self.minor_collections += 1;
}
if is_major {
self.major_collections += 1;
}
self.total_gc_time += pause_time;
self.total_bytes_freed += bytes_freed;
if pause_time > self.max_gc_pause_time {
self.max_gc_pause_time = pause_time;
}
let n = self.total_gc_cycles as f64;
self.avg_gc_pause_time = (self.avg_gc_pause_time * (n - 1.0) + pause_time as f64) / n;
if self.total_gc_time > 0 {
self.collection_efficiency = self.total_bytes_freed as f64 / self.total_gc_time as f64;
}
}
pub fn record_oom(&mut self) {
self.oom_events += 1;
}
pub fn summary(&self) -> String {
format!(
"GC Stats: {} cycles ({} minor, {} major), {:.1}ms avg pause, {}MB freed, efficiency: {:.1} MB/ms",
self.total_gc_cycles,
self.minor_collections,
self.major_collections,
self.avg_gc_pause_time,
self.total_bytes_freed / (1024 * 1024),
self.collection_efficiency,
)
}
pub fn reset(&mut self) {
*self = X86GCStatistics::new();
}
}
#[derive(Debug, Clone)]
pub struct X86GCRememberedSet {
pub entries: HashSet<u64>,
pub max_size: usize,
pub overflowed: bool,
pub entries_since_clear: u64,
}
impl X86GCRememberedSet {
pub fn new(max_size: usize) -> Self {
X86GCRememberedSet {
entries: HashSet::new(),
max_size,
overflowed: false,
entries_since_clear: 0,
}
}
pub fn add(&mut self, address: u64) {
if self.overflowed {
return;
}
if self.entries.len() >= self.max_size {
self.overflowed = true;
return;
}
self.entries.insert(address);
self.entries_since_clear += 1;
}
pub fn contains(&self, address: u64) -> bool {
self.entries.contains(&address)
}
pub fn remove(&mut self, address: u64) -> bool {
self.entries.remove(&address)
}
pub fn clear(&mut self) {
self.entries.clear();
self.overflowed = false;
self.entries_since_clear = 0;
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
pub fn drain(&mut self) -> Vec<u64> {
let entries: Vec<u64> = self.entries.drain().collect();
self.overflowed = false;
self.entries_since_clear = 0;
entries
}
}
impl Default for X86GCRememberedSet {
fn default() -> Self {
X86GCRememberedSet::new(4096)
}
}
#[derive(Debug)]
pub struct X86GCFull {
pub strategy: X86GCStrategy,
pub stack_map_generator: X86GCStackMapGenerator,
pub stack_map_parser: X86GCStackMapParser,
pub safepoints: X86GCSafepoints,
pub roots: Vec<X86GCRoot>,
pub finalization: X86GCFinalization,
pub card_table: Option<X86GCCardTable>,
pub satb_buffer: Option<X86GCSATBBuffer>,
pub gc_enabled: AtomicBool,
pub gc_cycles: AtomicU64,
pub total_bytes_allocated: AtomicU64,
pub total_bytes_freed: AtomicU64,
pub mark_sweep_collector: Option<X86GCMarkSweepCollector>,
pub copying_collector: Option<X86GCCopyingCollector>,
pub generational_collector: Option<X86GCGenerationalCollector>,
pub incremental_collector: Option<X86GCIncrementalCollector>,
pub bump_allocator: Option<X86GCBumpAllocator>,
pub free_list_allocator: Option<X86GCFreeListAllocator>,
pub tlab: Option<X86GCTLAB>,
pub large_object_allocator: Option<X86GCLargeObjectAllocator>,
pub config: X86GCFullConfig,
pub stack_walker: Option<X86GCStackWalker>,
pub memory_pressure: X86GCMemoryPressure,
pub thread_suspension: X86GCThreadSuspension,
pub deoptimization: Option<X86GCDeoptimization>,
pub statistics: X86GCStatistics,
pub remembered_set: Option<Box<X86GCRememberedSet>>,
}
#[derive(Debug, Clone)]
pub struct X86GCFullConfig {
pub emit_stack_maps: bool,
pub insert_loop_polls: bool,
pub rewrite_statepoints: bool,
pub card_table_size: u64,
pub emit_write_barriers: bool,
pub emit_read_barriers: bool,
pub is_exact: bool,
pub poll_frequency: X86GCSafepointFrequency,
pub concurrent_gc: bool,
pub initial_heap_size: u64,
pub max_heap_size: u64,
pub nursery_size: u64,
pub use_generational: bool,
pub use_incremental: bool,
pub collection_algorithm: X86GCCollectionAlgorithm,
pub enable_finalization: bool,
pub process_weak_refs: bool,
pub process_ephemerons: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86GCCollectionAlgorithm {
MarkSweep,
MarkCompact,
Copying,
Generational,
Incremental,
None,
}
impl fmt::Display for X86GCCollectionAlgorithm {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
X86GCCollectionAlgorithm::MarkSweep => write!(f, "mark-sweep"),
X86GCCollectionAlgorithm::MarkCompact => write!(f, "mark-compact"),
X86GCCollectionAlgorithm::Copying => write!(f, "copying"),
X86GCCollectionAlgorithm::Generational => write!(f, "generational"),
X86GCCollectionAlgorithm::Incremental => write!(f, "incremental"),
X86GCCollectionAlgorithm::None => write!(f, "none"),
}
}
}
impl Default for X86GCFullConfig {
fn default() -> Self {
X86GCFullConfig {
emit_stack_maps: true,
insert_loop_polls: false,
rewrite_statepoints: true,
card_table_size: 1024 * 1024,
emit_write_barriers: false,
emit_read_barriers: false,
is_exact: true,
poll_frequency: X86GCSafepointFrequency::StatepointsOnly,
concurrent_gc: false,
initial_heap_size: 64 * 1024 * 1024,
max_heap_size: 1024 * 1024 * 1024,
nursery_size: X86_GC_DEFAULT_NURSERY_SIZE as u64,
use_generational: false,
use_incremental: false,
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
enable_finalization: true,
process_weak_refs: true,
process_ephemerons: false,
}
}
}
#[derive(Debug, Clone)]
pub struct X86GCCardTable {
pub cards: Vec<u8>,
pub heap_start: u64,
pub heap_end: u64,
pub card_shift: u8,
pub num_cards: u64,
}
impl X86GCCardTable {
pub fn new(heap_start: u64, heap_size: u64, card_shift: u8) -> Self {
let card_size = 1u64 << card_shift;
let num_cards = (heap_size + card_size - 1) / card_size;
X86GCCardTable {
cards: vec![0u8; num_cards as usize],
heap_start,
heap_end: heap_start + heap_size,
card_shift,
num_cards,
}
}
pub fn card_index(&self, address: u64) -> usize {
((address - self.heap_start) >> self.card_shift) as usize
}
pub fn mark_dirty(&mut self, address: u64) {
let idx = self.card_index(address);
if idx < self.cards.len() {
self.cards[idx] = 1;
}
}
pub fn clear_card(&mut self, address: u64) {
let idx = self.card_index(address);
if idx < self.cards.len() {
self.cards[idx] = 0;
}
}
pub fn is_dirty(&self, address: u64) -> bool {
let idx = self.card_index(address);
idx < self.cards.len() && self.cards[idx] != 0
}
pub fn clear_all(&mut self) {
for card in &mut self.cards {
*card = 0;
}
}
pub fn dirty_cards(&self) -> Vec<u64> {
self.cards
.iter()
.enumerate()
.filter(|&(_, &c)| c != 0)
.map(|(i, _)| self.heap_start + (i as u64) * (1u64 << self.card_shift))
.collect()
}
pub fn dirty_count(&self) -> usize {
self.cards.iter().filter(|&&c| c != 0).count()
}
}
impl Default for X86GCCardTable {
fn default() -> Self {
X86GCCardTable::new(0, 64 * 1024 * 1024, 9) }
}
#[derive(Debug, Clone)]
pub struct X86GCSATBBuffer {
pub entries: Vec<u64>,
pub index: u64,
pub capacity: u64,
pub overflowed: bool,
}
impl X86GCSATBBuffer {
pub fn new(capacity: u64) -> Self {
X86GCSATBBuffer {
entries: Vec::with_capacity(capacity as usize),
index: 0,
capacity,
overflowed: false,
}
}
pub fn log(&mut self, value: u64) {
if self.overflowed {
return;
}
if self.entries.len() < self.capacity as usize {
self.entries.push(value);
self.index += 1;
} else {
self.overflowed = true;
}
}
pub fn drain(&mut self) -> Vec<u64> {
let entries = std::mem::take(&mut self.entries);
self.index = 0;
self.overflowed = false;
entries
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
}
impl Default for X86GCSATBBuffer {
fn default() -> Self {
X86GCSATBBuffer::new(1024)
}
}
impl X86GCFull {
pub fn new(strategy: X86GCStrategy, config: X86GCFullConfig) -> Self {
let mut gc = X86GCFull {
strategy: strategy.clone(),
stack_map_generator: X86GCStackMapGenerator::new(),
stack_map_parser: X86GCStackMapParser::new(),
safepoints: X86GCSafepoints::default(),
roots: Vec::new(),
finalization: X86GCFinalization::new(),
card_table: None,
satb_buffer: None,
gc_enabled: AtomicBool::new(true),
gc_cycles: AtomicU64::new(0),
total_bytes_allocated: AtomicU64::new(0),
total_bytes_freed: AtomicU64::new(0),
mark_sweep_collector: None,
copying_collector: None,
generational_collector: None,
incremental_collector: None,
bump_allocator: None,
free_list_allocator: None,
tlab: None,
large_object_allocator: None,
config,
stack_walker: None,
memory_pressure: X86GCMemoryPressure::default(),
thread_suspension: X86GCThreadSuspension::default(),
deoptimization: None,
statistics: X86GCStatistics::default(),
remembered_set: None,
};
gc.initialize_collectors();
gc
}
fn initialize_collectors(&mut self) {
let heap_base = 0x10000u64; let heap_size = self.config.initial_heap_size;
match self.config.collection_algorithm {
X86GCCollectionAlgorithm::MarkSweep => {
self.mark_sweep_collector =
Some(X86GCMarkSweepCollector::new(heap_base, heap_size));
}
X86GCCollectionAlgorithm::Copying => {
let semispace = heap_size / 2;
self.copying_collector = Some(X86GCCopyingCollector::new(
heap_base,
heap_base + semispace,
semispace,
));
}
X86GCCollectionAlgorithm::Generational => {
let nursery_size = self.config.nursery_size;
let old_size = heap_size - nursery_size;
self.generational_collector = Some(X86GCGenerationalCollector::new(
heap_base,
nursery_size,
heap_base + nursery_size,
old_size,
));
}
X86GCCollectionAlgorithm::Incremental => {
self.incremental_collector =
Some(X86GCIncrementalCollector::new(heap_base, heap_size));
}
X86GCCollectionAlgorithm::MarkCompact | X86GCCollectionAlgorithm::None => {}
}
if self.strategy.requires_write_barriers()
|| self.config.collection_algorithm == X86GCCollectionAlgorithm::Generational
{
self.card_table = Some(X86GCCardTable::new(heap_base, heap_size, 9));
}
if self.config.concurrent_gc
|| self.config.collection_algorithm == X86GCCollectionAlgorithm::Incremental
{
self.satb_buffer = Some(X86GCSATBBuffer::new(1024));
}
self.bump_allocator = Some(X86GCBumpAllocator::new(
heap_base,
heap_size / 4,
X86_GC_OBJECT_ALIGNMENT,
));
self.free_list_allocator = Some(X86GCFreeListAllocator::new(heap_size));
self.large_object_allocator = Some(X86GCLargeObjectAllocator::new(
X86_GC_LARGE_OBJECT_THRESHOLD as u64,
));
}
pub fn begin_function(&mut self, name: &str, address: u64) {
self.stack_map_generator.begin_function(name, address);
self.safepoints
.insert_entry_safepoint(name, &mut self.stack_map_generator);
}
pub fn end_function(&mut self) {
self.stack_map_generator.end_function();
}
pub fn add_root(&mut self, root: X86GCRoot) {
self.roots.push(root);
}
pub fn add_safepoint(
&mut self,
function_name: &str,
offset: u64,
kind: X86GCSafepointKind,
) -> Option<u64> {
match kind {
X86GCSafepointKind::LoopBackedge => self.safepoints.insert_backedge_safepoint(
function_name,
offset,
&mut self.stack_map_generator,
),
X86GCSafepointKind::AfterCall => self.safepoints.insert_call_safepoint(
function_name,
offset,
&mut self.stack_map_generator,
),
X86GCSafepointKind::FunctionEntry => self
.safepoints
.insert_entry_safepoint(function_name, &mut self.stack_map_generator),
_ => None,
}
}
pub fn record_write_barrier(&mut self, object_base: u64, field_offset: u64, new_value: u64) {
if let Some(ref mut card_table) = self.card_table {
card_table.mark_dirty(object_base);
}
let _ = (field_offset, new_value);
}
pub fn record_satb_barrier(&mut self, old_value: u64) {
if let Some(ref mut satb) = self.satb_buffer {
satb.log(old_value);
}
}
pub fn allocate(&mut self, size: u64) -> u64 {
if let Some(ref mut tlab) = self.tlab {
let result = tlab.allocate_fast(size);
if result != 0 {
self.total_bytes_allocated
.fetch_add(size, Ordering::Relaxed);
return result;
}
}
if let Some(ref mut bump) = self.bump_allocator {
let result = bump.allocate_fast(size);
if result != 0 {
self.total_bytes_allocated
.fetch_add(size, Ordering::Relaxed);
return result;
}
}
if let Some(ref mut fl) = self.free_list_allocator {
let result = fl.allocate(size as u32);
if result != 0 {
self.total_bytes_allocated
.fetch_add(size, Ordering::Relaxed);
return result;
}
}
if let Some(ref mut lo) = self.large_object_allocator {
let result = lo.allocate(size);
if result != 0 {
self.total_bytes_allocated
.fetch_add(size, Ordering::Relaxed);
return result;
}
}
self.collect();
if let Some(ref mut bump) = self.bump_allocator {
let result = bump.allocate_fast(size);
if result != 0 {
self.total_bytes_allocated
.fetch_add(size, Ordering::Relaxed);
return result;
}
}
0 }
pub fn collect(&mut self) {
self.gc_enabled.store(false, Ordering::SeqCst);
match self.config.collection_algorithm {
X86GCCollectionAlgorithm::MarkSweep => {
if let Some(ref mut ms) = self.mark_sweep_collector {
ms.collect(&self.roots);
}
}
X86GCCollectionAlgorithm::Copying => {
if let Some(ref mut cp) = self.copying_collector {
let root_addrs: Vec<u64> = self
.roots
.iter()
.filter(|r| r.is_live)
.map(|_| 0u64) .collect();
cp.collect(&root_addrs);
}
}
X86GCCollectionAlgorithm::Generational => {
if let Some(ref mut r#gen) = self.generational_collector {
let root_addrs: Vec<u64> = Vec::new();
r#gen.collect(&self.roots, &root_addrs);
}
}
X86GCCollectionAlgorithm::Incremental => {
if let Some(ref mut inc) = self.incremental_collector {
let root_addrs: Vec<u64> = Vec::new();
inc.start_mark(&root_addrs);
}
}
X86GCCollectionAlgorithm::MarkCompact | X86GCCollectionAlgorithm::None => {}
}
if self.config.enable_finalization {
self.finalization
.process_finalizers(X86_GC_MAX_FINALIZERS_PER_CYCLE);
}
if let Some(ref mut ct) = self.card_table {
ct.clear_all();
}
if let Some(ref mut satb) = self.satb_buffer {
let _entries = satb.drain();
}
self.gc_cycles.fetch_add(1, Ordering::SeqCst);
self.gc_enabled.store(true, Ordering::SeqCst);
}
pub fn incremental_marking_step(&mut self, steps: usize) -> bool {
if let Some(ref mut inc) = self.incremental_collector {
inc.marking_step(steps)
} else {
true
}
}
pub fn incremental_sweeping_step(&mut self) -> bool {
if let Some(ref mut inc) = self.incremental_collector {
inc.sweeping_step()
} else {
true
}
}
pub fn finalize_stack_maps(&mut self) -> X86GCStackMap {
self.stack_map_generator.finalize()
}
pub fn emit_stack_map_bytes(&self) -> Vec<u8> {
self.stack_map_generator.stack_map.emit_bytes()
}
pub fn emit_stack_map_assembly(&self) -> String {
self.stack_map_generator.stack_map.emit_assembly()
}
pub fn enable(&self) {
self.gc_enabled.store(true, Ordering::SeqCst);
}
pub fn disable(&self) {
self.gc_enabled.store(false, Ordering::SeqCst);
}
pub fn is_enabled(&self) -> bool {
self.gc_enabled.load(Ordering::SeqCst)
}
pub fn gc_cycles(&self) -> u64 {
self.gc_cycles.load(Ordering::SeqCst)
}
pub fn total_allocated(&self) -> u64 {
self.total_bytes_allocated.load(Ordering::SeqCst)
}
pub fn validate(&self) -> Result<(), String> {
if self.roots.is_empty()
&& self.config.collection_algorithm != X86GCCollectionAlgorithm::None
{
return Err("No roots registered for GC collection".to_string());
}
if self.config.emit_stack_maps {
if self.stack_map_generator.stack_map.functions.is_empty() {
return Err("Stack maps enabled but no functions registered".to_string());
}
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct X86GCFrameDescriptor {
pub return_address: u64,
pub frame_size: u32,
pub num_roots: u16,
pub root_bitmap: u64,
pub is_valid: bool,
}
impl X86GCFrameDescriptor {
pub fn new(return_address: u64, frame_size: u32) -> Self {
X86GCFrameDescriptor {
return_address,
frame_size,
num_roots: 0,
root_bitmap: 0,
is_valid: true,
}
}
pub fn add_root(&mut self, slot_index: u8) {
if slot_index < 64 {
self.root_bitmap |= 1u64 << slot_index;
self.num_roots += 1;
}
}
pub fn is_root(&self, slot_index: u8) -> bool {
if slot_index >= 64 {
return false;
}
(self.root_bitmap & (1u64 << slot_index)) != 0
}
pub fn root_offsets(&self) -> Vec<i32> {
(0..64)
.filter(|&i| self.is_root(i))
.map(|i| -((i as i32 + 1) * 8)) .collect()
}
}
#[derive(Debug, Clone)]
pub struct X86GCOCamlTable {
pub descriptors: HashMap<u64, X86GCFrameDescriptor>,
pub addr_list: Vec<u64>,
}
impl X86GCOCamlTable {
pub fn new() -> Self {
X86GCOCamlTable {
descriptors: HashMap::new(),
addr_list: Vec::new(),
}
}
pub fn add_descriptor(&mut self, desc: X86GCFrameDescriptor) {
self.addr_list.push(desc.return_address);
self.descriptors.insert(desc.return_address, desc);
}
pub fn lookup(&self, return_address: u64) -> Option<&X86GCFrameDescriptor> {
self.descriptors.get(&return_address)
}
pub fn len(&self) -> usize {
self.descriptors.len()
}
pub fn is_empty(&self) -> bool {
self.descriptors.is_empty()
}
}
impl Default for X86GCOCamlTable {
fn default() -> Self {
X86GCOCamlTable::new()
}
}
#[derive(Debug, Clone)]
pub struct X86GCCoreCLRInfo {
pub version: u8,
pub return_address: u64,
pub method_size: u32,
pub is_fully_interruptible: bool,
pub interruptible_regions: Vec<X86GCCoreCLRRegion>,
pub live_slots: Vec<X86GCCoreCLRLiveSlot>,
pub transitions: Vec<X86GCCoreCLRTransition>,
pub has_security_object: bool,
pub has_gen_cookie: bool,
pub epilog_count: u8,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCCoreCLRRegion {
pub start_offset: u32,
pub end_offset: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCCoreCLRLiveSlot {
pub stack_offset: i32,
pub is_byref: bool,
pub is_pinned: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct X86GCCoreCLRTransition {
pub instruction_offset: u32,
pub code_offset: u32,
pub is_call_site: bool,
}
impl X86GCCoreCLRInfo {
pub fn new(return_address: u64, method_size: u32) -> Self {
X86GCCoreCLRInfo {
version: 2,
return_address,
method_size,
is_fully_interruptible: false,
interruptible_regions: Vec::new(),
live_slots: Vec::new(),
transitions: Vec::new(),
has_security_object: false,
has_gen_cookie: false,
epilog_count: 0,
}
}
pub fn mark_fully_interruptible(&mut self) {
self.is_fully_interruptible = true;
}
pub fn add_region(&mut self, start: u32, end: u32) {
self.interruptible_regions.push(X86GCCoreCLRRegion {
start_offset: start,
end_offset: end,
});
}
pub fn add_live_slot(&mut self, offset: i32, is_byref: bool, is_pinned: bool) {
self.live_slots.push(X86GCCoreCLRLiveSlot {
stack_offset: offset,
is_byref,
is_pinned,
});
}
pub fn add_transition(&mut self, inst_offset: u32, code_offset: u32, is_call: bool) {
self.transitions.push(X86GCCoreCLRTransition {
instruction_offset: inst_offset,
code_offset,
is_call_site: is_call,
});
}
pub fn encode(&self) -> Vec<u8> {
let mut buf = Vec::new();
buf.push(self.version);
buf.push(if self.is_fully_interruptible { 1 } else { 0 });
buf.push(self.epilog_count);
let mut flags: u8 = 0;
if self.has_security_object {
flags |= 1;
}
if self.has_gen_cookie {
flags |= 2;
}
buf.push(flags);
encode_uleb128(&mut buf, self.method_size as u64);
encode_uleb128(&mut buf, self.live_slots.len() as u64);
for slot in &self.live_slots {
encode_sleb128(&mut buf, slot.stack_offset as i64);
let slot_flags: u8 =
if slot.is_byref { 1 } else { 0 } | if slot.is_pinned { 2 } else { 0 };
buf.push(slot_flags);
}
encode_uleb128(&mut buf, self.transitions.len() as u64);
for t in &self.transitions {
encode_uleb128(&mut buf, t.instruction_offset as u64);
encode_uleb128(&mut buf, t.code_offset as u64);
buf.push(if t.is_call_site { 1 } else { 0 });
}
buf
}
}
fn encode_uleb128(buf: &mut Vec<u8>, mut value: u64) {
loop {
let mut byte = (value & 0x7F) as u8;
value >>= 7;
if value != 0 {
byte |= 0x80;
}
buf.push(byte);
if value == 0 {
break;
}
}
}
fn encode_sleb128(buf: &mut Vec<u8>, mut value: i64) {
loop {
let mut byte = (value & 0x7F) as u8;
value >>= 7;
if (value == 0 && (byte & 0x40) == 0) || (value == -1 && (byte & 0x40) != 0) {
buf.push(byte);
break;
}
byte |= 0x80;
buf.push(byte);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_strategy_names() {
assert_eq!(X86GCStrategy::ShadowStack.name(), "shadow-stack");
assert_eq!(X86GCStrategy::Statepoint.name(), "statepoint");
assert_eq!(X86GCStrategy::CoreCLR.name(), "coreclr");
assert_eq!(X86GCStrategy::Erlang.name(), "erlang");
assert_eq!(X86GCStrategy::OCaml.name(), "ocaml");
}
#[test]
fn test_strategy_uses_statepoints() {
assert!(X86GCStrategy::Statepoint.uses_statepoints());
assert!(!X86GCStrategy::ShadowStack.uses_statepoints());
assert!(!X86GCStrategy::CoreCLR.uses_statepoints());
}
#[test]
fn test_strategy_supports_generational() {
assert!(X86GCStrategy::ShadowStack.supports_generational());
assert!(X86GCStrategy::Statepoint.supports_generational());
assert!(!X86GCStrategy::OCaml.supports_generational());
}
#[test]
fn test_strategy_requires_write_barriers() {
assert!(X86GCStrategy::CoreCLR.requires_write_barriers());
assert!(!X86GCStrategy::Statepoint.requires_write_barriers());
}
#[test]
fn test_custom_strategy_default() {
let custom = X86GCCustomStrategy::default();
assert_eq!(custom.name, "custom");
assert!(!custom.uses_statepoints);
assert!(!custom.uses_shadow_stack);
assert_eq!(custom.suspension, X86GCSuspension::Polling);
}
#[test]
fn test_strategy_display() {
assert_eq!(format!("{}", X86GCStrategy::ShadowStack), "shadow-stack");
assert_eq!(format!("{}", X86GCStrategy::Erlang), "erlang");
}
#[test]
fn test_strategy_default() {
let default = X86GCStrategy::default();
assert_eq!(default, X86GCStrategy::Statepoint);
}
#[test]
fn test_suspension_display() {
assert_eq!(format!("{}", X86GCSuspension::Polling), "polling");
assert_eq!(format!("{}", X86GCSuspension::None), "none");
assert_eq!(format!("{}", X86GCSuspension::PageFault), "pagefault");
}
#[test]
fn test_root_new_base() {
let root = X86GCRoot::new_base(1, -8);
assert_eq!(root.id, 1);
assert_eq!(root.frame_offset, -8);
assert!(!root.is_derived);
assert!(root.is_live);
assert_eq!(root.pointer_size, 8);
}
#[test]
fn test_root_new_derived() {
let root = X86GCRoot::new_derived(2, -16, 1, 24);
assert_eq!(root.id, 2);
assert!(root.is_derived);
assert_eq!(root.base_root_id, Some(1));
assert_eq!(root.interior_offset, 24);
}
#[test]
fn test_root_new_register() {
let root = X86GCRoot::new_register(3, DWARF_REG_RAX);
assert_eq!(root.id, 3);
assert_eq!(root.register, Some(DWARF_REG_RAX));
}
#[test]
fn test_root_new_static() {
let root = X86GCRoot::new_static(4, "my_global".to_string());
assert!(root.is_constant);
assert_eq!(root.symbol_name, Some("my_global".to_string()));
}
#[test]
fn test_root_new_thread_local() {
let root = X86GCRoot::new_thread_local(5, 42);
assert_eq!(root.tls_index, Some(42));
}
#[test]
fn test_root_with_type_metadata() {
let root = X86GCRoot::new_base(1, -8)
.with_type_metadata(X86GCRootType::PointerArray { length: 10 });
assert!(root.type_metadata.is_some());
assert_eq!(root.slot_size(), 80);
}
#[test]
fn test_root_with_language() {
let root = X86GCRoot::new_base(1, -8).with_language(X86GCRootLanguage::Rust);
assert_eq!(root.source_language, X86GCRootLanguage::Rust);
}
#[test]
fn test_root_is_live_at() {
let root = X86GCRoot::new_base(1, -8).with_live_range(10, 100);
assert!(root.is_live_at(50));
assert!(!root.is_live_at(5));
assert!(!root.is_live_at(200));
}
#[test]
fn test_root_mark_dead() {
let root = X86GCRoot::new_base(1, -8).mark_dead();
assert!(!root.is_live);
}
#[test]
fn test_root_needs_conservative_scan() {
let opaque =
X86GCRoot::new_base(1, -8).with_type_metadata(X86GCRootType::Opaque { size: 64 });
assert!(opaque.needs_conservative_scan());
let precise = X86GCRoot::new_base(2, -16);
assert!(!precise.needs_conservative_scan());
}
#[test]
fn test_root_type_has_pointers() {
assert!(X86GCRootType::Pointer.has_pointers());
assert!(!X86GCRootType::Leaf.has_pointers());
assert!(X86GCRootType::WeakPointer.has_pointers());
}
#[test]
fn test_root_type_is_precise() {
assert!(X86GCRootType::Pointer.is_precise());
assert!(!X86GCRootType::Opaque { size: 32 }.is_precise());
}
#[test]
fn test_barrier_card_marking() {
let barrier = X86GCBarrier::new_card_marking(0x1000, 0x10, 0x2000);
assert_eq!(barrier.barrier_type, X86GCBarrierType::Write);
assert_eq!(
barrier.write_barrier_kind,
X86GCWriteBarrierKind::CardMarking
);
assert_eq!(barrier.object_base, 0x1000);
assert_eq!(barrier.field_offset, 0x10);
}
#[test]
fn test_barrier_satb() {
let barrier = X86GCBarrier::new_satb(0x1000, 0x10, 0xDEAD);
assert_eq!(barrier.write_barrier_kind, X86GCWriteBarrierKind::SATB);
assert_eq!(barrier.old_value, Some(0xDEAD));
}
#[test]
fn test_barrier_brooks_read() {
let barrier = X86GCBarrier::new_brooks_read(0x3000);
assert_eq!(barrier.barrier_type, X86GCBarrierType::Read);
assert_eq!(barrier.read_barrier_kind, X86GCReadBarrierKind::Brooks);
}
#[test]
fn test_barrier_acquire_load() {
let barrier = X86GCBarrier::new_acquire_load(0x4000);
assert_eq!(barrier.read_barrier_kind, X86GCReadBarrierKind::AcquireLoad);
}
#[test]
fn test_barrier_mark_array_element() {
let barrier = X86GCBarrier::new_card_marking(0x1000, 0x20, 0x3000).mark_array_element();
assert!(barrier.is_array_element);
}
#[test]
fn test_barrier_emit_card_marking_asm() {
let barrier = X86GCBarrier::new_card_marking(0x1000, 0x8, 0x42);
let code = barrier.emit_x86_card_marking_asm(0x8000, 9);
assert!(!code.is_empty());
assert!(code.len() > 8);
}
#[test]
fn test_barrier_emit_satb_asm() {
let barrier = X86GCBarrier::new_satb(0x1000, 0x8, 0xABCD);
let code = barrier.emit_x86_satb_asm(0x9000, 0xA000);
assert!(!code.is_empty());
}
#[test]
fn test_barrier_emit_brooks_read_asm() {
let barrier = X86GCBarrier::new_brooks_read(0x5000);
let code = barrier.emit_x86_brooks_read_asm();
assert!(!code.is_empty());
assert!(code.len() >= 10);
}
#[test]
fn test_write_barrier_kind_display() {
assert_eq!(
format!("{}", X86GCWriteBarrierKind::CardMarking),
"card-marking"
);
assert_eq!(format!("{}", X86GCWriteBarrierKind::SATB), "satb");
assert_eq!(format!("{}", X86GCWriteBarrierKind::None), "none");
}
#[test]
fn test_bump_allocator_allocate() {
let mut alloc = X86GCBumpAllocator::new(0x1000, 4096, 8);
let addr1 = alloc.allocate_fast(64);
assert_ne!(addr1, 0);
assert_eq!(addr1, 0x1000);
let addr2 = alloc.allocate_fast(128);
assert_eq!(addr2, 0x1000 + 64);
assert_eq!(alloc.allocation_count, 2);
}
#[test]
fn test_bump_allocator_exhaustion() {
let mut alloc = X86GCBumpAllocator::new(0x1000, 256, 8);
let addr = alloc.allocate_fast(300);
assert_eq!(addr, 0);
assert_eq!(alloc.slow_path_count, 1);
}
#[test]
fn test_bump_allocator_alignment() {
let mut alloc = X86GCBumpAllocator::new(0x1000, 4096, 16);
let addr = alloc.allocate_fast(13);
assert_eq!(addr % 16, 0);
assert_eq!(alloc.bytes_allocated, 16); }
#[test]
fn test_bump_allocator_reset() {
let mut alloc = X86GCBumpAllocator::new(0x1000, 4096, 8);
let _ = alloc.allocate_fast(64);
alloc.reset();
assert_eq!(alloc.cursor, alloc.base);
assert_eq!(alloc.remaining(), 4096);
}
#[test]
fn test_bump_allocator_remaining() {
let mut alloc = X86GCBumpAllocator::new(0x1000, 4096, 8);
assert_eq!(alloc.remaining(), 4096);
let _ = alloc.allocate_fast(1024);
assert_eq!(alloc.remaining(), 4096 - 1024);
}
#[test]
fn test_free_list_allocate_exact() {
let mut alloc = X86GCFreeListAllocator::new(1024 * 1024);
alloc.add_free_block(0x1000, 64);
let addr = alloc.allocate(64);
assert_eq!(addr, 0x1000);
}
#[test]
fn test_free_list_allocate_split() {
let mut alloc = X86GCFreeListAllocator::new(1024 * 1024);
alloc.add_free_block(0x1000, 128);
let addr = alloc.allocate(64);
assert_eq!(addr, 0x1000);
assert!(alloc.free_bytes() >= 64);
}
#[test]
fn test_free_list_allocate_no_fit() {
let mut alloc = X86GCFreeListAllocator::new(1024 * 1024);
let addr = alloc.allocate(1024);
assert_eq!(addr, 0);
}
#[test]
fn test_free_list_free_and_allocate() {
let mut alloc = X86GCFreeListAllocator::new(1024 * 1024);
alloc.add_free_block(0x1000, 256);
let addr = alloc.allocate(128);
assert_eq!(addr, 0x1000);
alloc.free(addr, 128);
let addr2 = alloc.allocate(128);
assert_eq!(addr2, 0x1000);
}
#[test]
fn test_free_list_size_class() {
assert_eq!(X86GCFreeListAllocator::size_class(8), 16);
assert_eq!(X86GCFreeListAllocator::size_class(32), 32);
assert_eq!(X86GCFreeListAllocator::size_class(100), 128);
assert_eq!(X86GCFreeListAllocator::size_class(500), 512);
assert_eq!(X86GCFreeListAllocator::size_class(5000), 8192);
}
#[test]
fn test_tlab_allocate() {
let mut tlab = X86GCTLAB::new(1, 0x1000, 1024);
let addr = tlab.allocate_fast(64);
assert_ne!(addr, 0);
assert_eq!(tlab.bytes_allocated, 64);
}
#[test]
fn test_tlab_exhaustion() {
let mut tlab = X86GCTLAB::new(1, 0x1000, 128);
let _ = tlab.allocate_fast(64);
let addr = tlab.allocate_fast(128); assert_eq!(addr, 0);
}
#[test]
fn test_tlab_refill() {
let mut tlab = X86GCTLAB::new(1, 0x1000, 128);
assert_eq!(tlab.refill_count, 0);
tlab.refill(0x2000, 256);
assert_eq!(tlab.refill_count, 1);
assert_eq!(tlab.base, 0x2000);
assert!(tlab.is_active);
}
#[test]
fn test_tlab_retire() {
let mut tlab = X86GCTLAB::new(1, 0x1000, 256);
assert!(tlab.is_active);
tlab.retire();
assert!(!tlab.is_active);
}
#[test]
fn test_tlab_needs_refill() {
let mut tlab = X86GCTLAB::new(1, 0x1000, 256);
assert!(!tlab.needs_refill(128));
assert!(tlab.needs_refill(512));
}
#[test]
fn test_large_object_allocate() {
let mut lo = X86GCLargeObjectAllocator::new(85 * 1024);
let addr = lo.allocate(100 * 1024);
assert_ne!(addr, 0);
assert_eq!(lo.count, 1);
assert!(lo.contains(addr));
}
#[test]
fn test_large_object_below_threshold() {
let mut lo = X86GCLargeObjectAllocator::new(85 * 1024);
let addr = lo.allocate(64);
assert_eq!(addr, 0);
}
#[test]
fn test_large_object_free() {
let mut lo = X86GCLargeObjectAllocator::new(85 * 1024);
let addr = lo.allocate(100 * 1024);
assert!(lo.free(addr));
assert!(!lo.contains(addr));
assert_eq!(lo.count, 0);
}
#[test]
fn test_large_object_find_enclosing() {
let mut lo = X86GCLargeObjectAllocator::new(85 * 1024);
let addr = lo.allocate(200 * 1024);
let interior = addr + 1000;
let found = lo.find_enclosing_object(interior);
assert!(found.is_some());
assert_eq!(found.unwrap().0, addr);
}
#[test]
fn test_mark_sweep_new() {
let ms = X86GCMarkSweepCollector::new(0x10000, 1024 * 1024);
assert_eq!(ms.heap_base, 0x10000);
assert_eq!(ms.heap_size, 1024 * 1024);
assert_eq!(ms.collections, 0);
assert!(!ms.collecting);
}
#[test]
fn test_mark_sweep_set_get_mark() {
let mut ms = X86GCMarkSweepCollector::new(0x10000, 1024 * 1024);
ms.set_mark(0x10000);
assert!(ms.is_marked(0x10000));
assert!(!ms.is_marked(0x10008));
ms.set_mark(0x10008);
assert!(ms.is_marked(0x10008));
}
#[test]
fn test_mark_sweep_mark_out_of_range() {
let mut ms = X86GCMarkSweepCollector::new(0x10000, 4096);
ms.set_mark(0x20000); assert!(!ms.is_marked(0x20000));
}
#[test]
fn test_mark_sweep_collect() {
let mut ms = X86GCMarkSweepCollector::new(0x10000, 1024 * 1024);
let roots = vec![X86GCRoot::new_base(1, -8)];
let freed = ms.collect(&roots);
assert_eq!(ms.collections, 1);
let _ = freed;
}
#[test]
fn test_copying_new() {
let cp = X86GCCopyingCollector::new(0x10000, 0x40000, 0x30000);
assert_eq!(cp.fromspace_base, 0x10000);
assert_eq!(cp.tospace_base, 0x40000);
assert_eq!(cp.semispace_size, 0x30000);
}
#[test]
fn test_copying_allocate_fast() {
let mut cp = X86GCCopyingCollector::new(0x10000, 0x40000, 0x30000);
let addr = cp.allocate_fast(64);
assert_eq!(addr, 0x10000);
let addr2 = cp.allocate_fast(128);
assert_eq!(addr2, 0x10040);
}
#[test]
fn test_copying_forward() {
let mut cp = X86GCCopyingCollector::new(0x10000, 0x40000, 0x30000);
cp.free = 0x40000; let new_addr = cp.forward(0x10050, 64);
assert_eq!(new_addr, 0x40000);
let again = cp.forward(0x10050, 64);
assert_eq!(again, 0x40000);
}
#[test]
fn test_copying_collect() {
let mut cp = X86GCCopyingCollector::new(0x10000, 0x40000, 0x30000);
cp.free = 0x10080; let roots: Vec<u64> = vec![0x10000];
let copied = cp.collect(&roots);
assert_eq!(cp.collections, 1);
let _ = copied;
}
#[test]
fn test_generational_new() {
let r#gen = X86GCGenerationalCollector::new(0x10000, 1024 * 1024, 0x200000, 8 * 1024 * 1024);
assert_eq!(r#gen.minor_collections, 0);
assert_eq!(r#gen.major_collections, 0);
}
#[test]
fn test_generational_nursery_allocate() {
let mut r#gen =
X86GCGenerationalCollector::new(0x10000, 1024 * 1024, 0x200000, 8 * 1024 * 1024);
let addr = r#gen.nursery_allocate(128);
assert_eq!(addr, 0x10000);
}
#[test]
fn test_generational_record_write() {
let mut r#gen =
X86GCGenerationalCollector::new(0x10000, 1024 * 1024, 0x200000, 8 * 1024 * 1024);
r#gen.record_write(0x200100); assert!(r#gen.remembered_set.contains(&0x200100));
}
#[test]
fn test_generational_minor_collect() {
let mut r#gen =
X86GCGenerationalCollector::new(0x10000, 1024 * 1024, 0x200000, 8 * 1024 * 1024);
let roots = vec![X86GCRoot::new_base(1, -8)];
let root_vals = vec![0x10040u64];
r#gen.minor_collect(&roots, &root_vals);
assert_eq!(r#gen.minor_collections, 1);
}
#[test]
fn test_incremental_new() {
let inc = X86GCIncrementalCollector::new(0x10000, 1024 * 1024);
assert_eq!(inc.collections, 0);
assert!(!inc.marking);
assert!(!inc.sweeping);
}
#[test]
fn test_incremental_color() {
let mut inc = X86GCIncrementalCollector::new(0x10000, 1024 * 1024);
assert_eq!(inc.color(0x10000), X86GCTriColor::White);
inc.set_color(0x10000, X86GCTriColor::Gray);
assert_eq!(inc.color(0x10000), X86GCTriColor::Gray);
inc.set_color(0x10000, X86GCTriColor::Black);
assert_eq!(inc.color(0x10000), X86GCTriColor::Black);
}
#[test]
fn test_incremental_marking_step() {
let mut inc = X86GCIncrementalCollector::new(0x10000, 1024 * 1024);
inc.start_mark(&[0x10000, 0x10040]);
assert!(inc.marking);
let complete = inc.marking_step(10);
assert!(complete);
assert!(!inc.marking);
assert_eq!(inc.collections, 1);
}
#[test]
fn test_incremental_write_barrier() {
let mut inc = X86GCIncrementalCollector::new(0x10000, 1024 * 1024);
inc.start_mark(&[0x10040]);
inc.set_color(0x10040, X86GCTriColor::Black);
inc.write_barrier(0x10040, 0x10080);
assert_eq!(inc.color(0x10080), X86GCTriColor::Gray);
assert_eq!(inc.write_barriers_applied, 1);
}
#[test]
fn test_incremental_sweeping_step() {
let mut inc = X86GCIncrementalCollector::new(0x10000, 64 * 1024);
inc.sweeping = true;
inc.sweep_cursor = 0x10000;
let complete = inc.sweeping_step();
assert!(complete); }
#[test]
fn test_stack_map_new() {
let sm = X86GCStackMap::new();
assert_eq!(sm.version, 3);
assert_eq!(sm.functions.len(), 0);
assert_eq!(sm.records.len(), 0);
}
#[test]
fn test_stack_map_add_function() {
let mut sm = X86GCStackMap::new();
let idx = sm.add_function(0x400000, 128);
assert_eq!(idx, 0);
assert_eq!(sm.functions.len(), 1);
assert_eq!(sm.functions[0].function_address, 0x400000);
assert_eq!(sm.functions[0].stack_size, 128);
}
#[test]
fn test_stack_map_add_record() {
let mut sm = X86GCStackMap::new();
let func_idx = sm.add_function(0x400000, 128);
let record = X86GCStackMapRecord {
id: 1,
instruction_offset: 0x10,
flags: 0,
locations: vec![X86GCStackMapLocation::new_register(DWARF_REG_RBP, 8)],
live_outs: vec![],
function_index: func_idx,
};
sm.add_record(record, func_idx);
assert_eq!(sm.records.len(), 1);
assert_eq!(sm.functions[0].record_count, 1);
}
#[test]
fn test_stack_map_emit_bytes() {
let mut sm = X86GCStackMap::new();
let func_idx = sm.add_function(0x400000, 128);
let record = X86GCStackMapRecord {
id: 1,
instruction_offset: 0x10,
flags: 0,
locations: vec![],
live_outs: vec![],
function_index: func_idx,
};
sm.add_record(record, func_idx);
let bytes = sm.emit_bytes();
assert!(!bytes.is_empty());
assert!(bytes.len() >= 16); }
#[test]
fn test_stack_map_emit_assembly() {
let mut sm = X86GCStackMap::new();
sm.add_function(0x400000, 128);
let asm = sm.emit_assembly();
assert!(asm.contains(X86_GC_STACK_MAP_SECTION));
assert!(asm.contains("version"));
}
#[test]
fn test_location_register() {
let loc = X86GCStackMapLocation::new_register(DWARF_REG_RAX, 8);
assert_eq!(loc.kind, X86GCStackMapLocationKind::Register);
assert_eq!(loc.dwarf_reg, DWARF_REG_RAX);
}
#[test]
fn test_location_direct() {
let loc = X86GCStackMapLocation::new_direct(-8, 8);
assert_eq!(loc.kind, X86GCStackMapLocationKind::Direct);
assert_eq!(loc.offset, -8);
}
#[test]
fn test_location_kind_from_u8() {
assert_eq!(
X86GCStackMapLocationKind::from_u8(1),
Some(X86GCStackMapLocationKind::Register)
);
assert_eq!(
X86GCStackMapLocationKind::from_u8(2),
Some(X86GCStackMapLocationKind::Direct)
);
assert_eq!(X86GCStackMapLocationKind::from_u8(0), None);
assert_eq!(X86GCStackMapLocationKind::from_u8(6), None);
}
#[test]
fn test_generator_begin_end_function() {
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test_func", 0x400000);
assert_eq!(r#gen.current_function_name, Some("test_func".to_string()));
assert!(r#gen.current_function_index.is_some());
r#gen.end_function();
assert!(r#gen.current_function_name.is_none());
}
#[test]
fn test_generator_generate_record() {
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test", 0x400000);
r#gen.live_var_state.record_register("ptr1", DWARF_REG_RBX, 8);
r#gen.live_var_state.mark_gc_pointer("ptr1");
r#gen.advance_offset(0x20);
let id = r#gen.generate_record(true);
assert!(id > 0);
assert_eq!(r#gen.stack_map.records.len(), 1);
assert_eq!(r#gen.stack_map.records[0].flags, 1); }
#[test]
fn test_generator_generate_for_patchpoint() {
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test", 0x400000);
r#gen.live_var_state.record_register("ptr1", DWARF_REG_R12, 8);
r#gen.live_var_state.mark_gc_pointer("ptr1");
r#gen.advance_offset(0x30);
let id = r#gen.generate_for_patchpoint(100, "target_fn");
assert!(id > 0);
}
#[test]
fn test_generator_generate_for_statepoint() {
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test", 0x400000);
r#gen.live_var_state.record_stack("root1", -16, 8);
r#gen.live_var_state.mark_gc_pointer("root1");
r#gen.advance_offset(0x40);
let id = r#gen.generate_for_statepoint(200, "callee_fn", &["root1"]);
assert!(id > 0);
}
#[test]
fn test_generator_finalize() {
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test", 0x400000);
r#gen.end_function();
let sm = r#gen.finalize();
assert_eq!(sm.functions.len(), 1);
}
#[test]
fn test_live_var_state_record_register() {
let mut state = X86GCLiveVarState::new(256);
state.record_register("x", DWARF_REG_RAX, 8);
assert!(state.get_location("x").is_some());
}
#[test]
fn test_live_var_state_allocate_stack_slot() {
let mut state = X86GCLiveVarState::new(256);
let offset = state.allocate_stack_slot("y", 8);
assert!(offset < 0);
assert!(state.get_location("y").is_some());
}
#[test]
fn test_live_var_state_gc_pointers() {
let mut state = X86GCLiveVarState::new(256);
state.record_register("gc_ptr", DWARF_REG_RCX, 8);
state.mark_gc_pointer("gc_ptr");
assert!(state.is_gc_pointer("gc_ptr"));
let locs = state.get_gc_pointer_locations();
assert_eq!(locs.len(), 1);
}
#[test]
fn test_parser_new() {
let parser = X86GCStackMapParser::new();
assert!(!parser.parsed);
assert_eq!(parser.record_count(), 0);
}
#[test]
fn test_parser_parse_invalid() {
let mut parser = X86GCStackMapParser::new();
assert!(!parser.parse(&[0; 4]));
assert!(!parser.warnings.is_empty());
}
#[test]
fn test_parser_parse_valid_minimal() {
let mut parser = X86GCStackMapParser::new();
let mut sm = X86GCStackMap::new();
sm.add_function(0x400000, 128);
let bytes = sm.emit_bytes();
assert!(parser.parse(&bytes));
assert!(parser.parsed);
}
#[test]
fn test_parser_lookup_by_id() {
let mut sm = X86GCStackMap::new();
let fi = sm.add_function(0x400000, 128);
let rec = X86GCStackMapRecord {
id: 42,
instruction_offset: 0x10,
flags: 0,
locations: vec![],
live_outs: vec![],
function_index: fi,
};
sm.add_record(rec, fi);
let bytes = sm.emit_bytes();
let mut parser = X86GCStackMapParser::new();
parser.parse(&bytes);
let found = parser.lookup_by_id(42);
assert!(found.is_some());
assert_eq!(found.unwrap().id, 42);
}
#[test]
fn test_safepoints_new() {
let sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
assert!(sp.enabled);
assert_eq!(sp.polling_page_address, X86_GC_POLLING_PAGE_ADDR);
assert_eq!(sp.safepoint_count, 0);
}
#[test]
fn test_safepoints_insert_entry() {
let mut sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test_fn", 0x400000);
let id = sp.insert_entry_safepoint("test_fn", &mut r#gen);
assert!(id.is_some());
assert_eq!(sp.stats.entry_safepoints, 1);
}
#[test]
fn test_safepoints_insert_backedge() {
let mut sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test_fn", 0x400000);
let id = sp.insert_backedge_safepoint("test_fn", 0x20, &mut r#gen);
assert!(id.is_some());
assert_eq!(sp.stats.backedge_safepoints, 1);
}
#[test]
fn test_safepoints_disabled() {
let mut sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
sp.set_enabled(false);
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("test_fn", 0x400000);
let id = sp.insert_entry_safepoint("test_fn", &mut r#gen);
assert!(id.is_none());
}
#[test]
fn test_safepoints_generate_poll_bytes() {
let sp = X86GCSafepoints::new(0x7FFF_FFFF_FFFF_F000);
let bytes = sp.generate_poll_bytes();
assert!(!bytes.is_empty());
assert_eq!(bytes[0], 0x48);
assert_eq!(bytes[1], 0xA1);
}
#[test]
fn test_safepoints_should_insert_backedge() {
let mut sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
sp.configure(true, true, false, 4);
assert!(sp.should_insert_backedge(0)); assert!(sp.should_insert_backedge(4)); assert!(!sp.should_insert_backedge(1)); assert!(!sp.should_insert_backedge(3)); }
#[test]
fn test_safepoints_reset() {
let mut sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
let mut r#gen = X86GCStackMapGenerator::new();
r#gen.begin_function("fn", 0x400000);
sp.insert_entry_safepoint("fn", &mut r#gen);
sp.reset();
assert_eq!(sp.safepoint_count, 0);
assert_eq!(sp.stats.total(), 0);
}
#[test]
fn test_safepoints_record_statepoint() {
let mut sp = X86GCSafepoints::new(X86_GC_POLLING_PAGE_ADDR);
sp.record_statepoint("fn", 0x50, 99);
assert_eq!(sp.stats.statepoint_safepoints, 1);
let fn_sps = sp.get_function_safepoints("fn");
assert!(fn_sps.is_some());
assert_eq!(fn_sps.unwrap().len(), 1);
}
#[test]
fn test_finalization_new() {
let fin = X86GCFinalization::new();
assert!(fin.enabled);
assert_eq!(fin.pending_finalizer_count(), 0);
}
#[test]
fn test_finalization_register() {
let mut fin = X86GCFinalization::new();
fin.register_finalizer(0x1000, 0x2000, 0);
assert!(fin.has_finalizer(0x1000));
assert_eq!(fin.pending_finalizer_count(), 1);
}
#[test]
fn test_finalization_process() {
let mut fin = X86GCFinalization::new();
fin.register_finalizer(0x1000, 0x2000, 0);
fin.register_finalizer(0x2000, 0x3000, 1);
let processed = fin.process_finalizers(10);
assert_eq!(processed, 2);
assert_eq!(fin.total_finalizers_run, 2);
assert_eq!(fin.pending_finalizer_count(), 0);
}
#[test]
fn test_finalization_double_finalize_prevented() {
let mut fin = X86GCFinalization::new();
fin.register_finalizer(0x1000, 0x2000, 0);
fin.process_finalizers(10);
fin.register_finalizer(0x1000, 0x2000, 0);
let processed = fin.process_finalizers(10);
assert_eq!(processed, 0); }
#[test]
fn test_weak_reference_register_and_process() {
let mut fin = X86GCFinalization::new();
fin.register_weak_reference(0x5000, 0x6000, true);
let cleared = fin.process_weak_references(&|addr| addr != 0x6000);
assert_eq!(cleared, 1);
assert_eq!(fin.total_weak_cleared, 1);
}
#[test]
fn test_weak_reference_referent_live() {
let mut fin = X86GCFinalization::new();
fin.register_weak_reference(0x5000, 0x6000, true);
let cleared = fin.process_weak_references(&|addr| addr == 0x6000);
assert_eq!(cleared, 0);
}
#[test]
fn test_phantom_reference() {
let mut fin = X86GCFinalization::new();
fin.register_phantom_reference(0x7000, 0x8000);
fin.register_finalizer(0x8000, 0x9000, 0);
fin.process_finalizers(10);
let processed = fin.process_phantom_references(&|addr| addr != 0x8000);
assert_eq!(processed, 1);
}
#[test]
fn test_ephemeron() {
let mut fin = X86GCFinalization::new();
fin.register_ephemeron(0xA000, 0xB000);
let cleared = fin.process_ephemerons(&|addr| addr == 0xA000);
assert_eq!(cleared, 0);
fin.register_ephemeron(0xC000, 0xD000);
let cleared = fin.process_ephemerons(&|_| false);
assert_eq!(cleared, 1);
}
#[test]
fn test_finalization_cleanup() {
let mut fin = X86GCFinalization::new();
fin.register_weak_reference(0x5000, 0x6000, false);
fin.process_weak_references(&|_| false);
fin.cleanup();
assert!(fin.weak_references.is_empty());
}
#[test]
fn test_finalization_reset() {
let mut fin = X86GCFinalization::new();
fin.register_finalizer(0x1000, 0x2000, 0);
fin.register_weak_reference(0x3000, 0x4000, false);
fin.reset();
assert_eq!(fin.pending_finalizer_count(), 0);
assert!(fin.weak_references.is_empty());
}
#[test]
fn test_card_table_new() {
let ct = X86GCCardTable::new(0x10000, 1024 * 1024, 9);
assert!(!ct.is_dirty(0x10010));
assert_eq!(ct.dirty_count(), 0);
}
#[test]
fn test_card_table_mark_and_check() {
let mut ct = X86GCCardTable::new(0x10000, 1024 * 1024, 9);
ct.mark_dirty(0x10000);
assert!(ct.is_dirty(0x10000));
assert!(ct.is_dirty(0x10100)); assert_eq!(ct.dirty_count(), 1);
}
#[test]
fn test_card_table_clear_card() {
let mut ct = X86GCCardTable::new(0x10000, 1024 * 1024, 9);
ct.mark_dirty(0x10000);
ct.clear_card(0x10000);
assert!(!ct.is_dirty(0x10000));
}
#[test]
fn test_card_table_clear_all() {
let mut ct = X86GCCardTable::new(0x10000, 1024 * 1024, 9);
ct.mark_dirty(0x10000);
ct.mark_dirty(0x20000);
ct.clear_all();
assert_eq!(ct.dirty_count(), 0);
}
#[test]
fn test_card_table_dirty_cards() {
let mut ct = X86GCCardTable::new(0x10000, 1024 * 1024, 9);
ct.mark_dirty(0x10000);
ct.mark_dirty(0x20000);
let dirty = ct.dirty_cards();
assert_eq!(dirty.len(), 2);
}
#[test]
fn test_satb_buffer_log() {
let mut buf = X86GCSATBBuffer::new(1024);
buf.log(0xDEAD);
buf.log(0xBEEF);
assert_eq!(buf.entries.len(), 2);
assert!(!buf.overflowed);
}
#[test]
fn test_satb_buffer_overflow() {
let mut buf = X86GCSATBBuffer::new(2);
buf.log(1);
buf.log(2);
buf.log(3); assert!(buf.overflowed);
}
#[test]
fn test_satb_buffer_drain() {
let mut buf = X86GCSATBBuffer::new(1024);
buf.log(0x42);
let entries = buf.drain();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0], 0x42);
assert!(buf.is_empty());
}
#[test]
fn test_gc_full_new() {
let gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
assert!(gc.is_enabled());
assert_eq!(gc.gc_cycles(), 0);
}
#[test]
fn test_gc_full_begin_end_function() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
gc.begin_function("test", 0x400000);
assert_eq!(
gc.stack_map_generator.current_function_name,
Some("test".to_string())
);
gc.end_function();
assert!(gc.stack_map_generator.current_function_name.is_none());
}
#[test]
fn test_gc_full_add_root() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
gc.add_root(X86GCRoot::new_base(1, -8));
assert_eq!(gc.roots.len(), 1);
}
#[test]
fn test_gc_full_add_safepoint() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
gc.begin_function("test", 0x400000);
let id = gc.add_safepoint("test", 0x20, X86GCSafepointKind::LoopBackedge);
assert!(id.is_some());
}
#[test]
fn test_gc_full_record_write_barrier() {
let mut gc = X86GCFull::new(
X86GCStrategy::CoreCLR,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::Generational,
..Default::default()
},
);
gc.record_write_barrier(0x10000, 0x8, 0xDEAD);
if let Some(ref ct) = gc.card_table {
assert!(ct.is_dirty(0x10000));
}
}
#[test]
fn test_gc_full_collect() {
let mut gc = X86GCFull::new(
X86GCStrategy::Statepoint,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
initial_heap_size: 64 * 1024 * 1024,
..Default::default()
},
);
gc.add_root(X86GCRoot::new_base(1, -8));
gc.collect();
assert_eq!(gc.gc_cycles(), 1);
assert!(gc.is_enabled());
}
#[test]
fn test_gc_full_allocate() {
let mut gc = X86GCFull::new(
X86GCStrategy::Statepoint,
X86GCFullConfig {
initial_heap_size: 1024 * 1024,
..Default::default()
},
);
let addr = gc.allocate(64);
assert_ne!(addr, 0);
assert!(gc.total_allocated() >= 64);
}
#[test]
fn test_gc_full_disable_enable() {
let gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
assert!(gc.is_enabled());
gc.disable();
assert!(!gc.is_enabled());
gc.enable();
assert!(gc.is_enabled());
}
#[test]
fn test_gc_full_validate_empty_roots() {
let gc = X86GCFull::new(
X86GCStrategy::Statepoint,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
..Default::default()
},
);
let result = gc.validate();
assert!(result.is_err());
}
#[test]
fn test_gc_full_validate_with_roots() {
let mut gc = X86GCFull::new(
X86GCStrategy::Statepoint,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
..Default::default()
},
);
gc.add_root(X86GCRoot::new_base(1, -8));
let result = gc.validate();
assert!(result.is_ok());
}
#[test]
fn test_gc_full_finalize_stack_maps() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
gc.begin_function("test", 0x400000);
gc.end_function();
let sm = gc.finalize_stack_maps();
assert_eq!(sm.functions.len(), 1);
}
#[test]
fn test_gc_full_emit_stack_map_bytes() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
gc.begin_function("test", 0x400000);
gc.end_function();
let bytes = gc.emit_stack_map_bytes();
assert!(!bytes.is_empty());
}
#[test]
fn test_gc_full_emit_stack_map_assembly() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
gc.begin_function("test", 0x400000);
gc.end_function();
let asm = gc.emit_stack_map_assembly();
assert!(!asm.is_empty());
assert!(asm.contains(X86_GC_STACK_MAP_SECTION));
}
#[test]
fn test_frame_descriptor_new() {
let fd = X86GCFrameDescriptor::new(0x400100, 256);
assert_eq!(fd.return_address, 0x400100);
assert_eq!(fd.frame_size, 256);
assert_eq!(fd.num_roots, 0);
assert!(fd.is_valid);
}
#[test]
fn test_frame_descriptor_add_root() {
let mut fd = X86GCFrameDescriptor::new(0x400100, 256);
fd.add_root(0);
fd.add_root(3);
assert!(fd.is_root(0));
assert!(fd.is_root(3));
assert!(!fd.is_root(1));
assert_eq!(fd.num_roots, 2);
}
#[test]
fn test_frame_descriptor_root_offsets() {
let mut fd = X86GCFrameDescriptor::new(0x400100, 256);
fd.add_root(0);
fd.add_root(2);
let offsets = fd.root_offsets();
assert_eq!(offsets.len(), 2);
assert!(offsets.contains(&-8));
assert!(offsets.contains(&-24));
}
#[test]
fn test_frame_descriptor_out_of_range() {
let mut fd = X86GCFrameDescriptor::new(0x400100, 256);
fd.add_root(63); assert!(fd.is_root(63));
fd.add_root(100); assert!(!fd.is_root(100));
}
#[test]
fn test_ocaml_table_add_and_lookup() {
let mut table = X86GCOCamlTable::new();
let fd = X86GCFrameDescriptor::new(0x400100, 256);
table.add_descriptor(fd);
assert_eq!(table.len(), 1);
let found = table.lookup(0x400100);
assert!(found.is_some());
assert_eq!(found.unwrap().frame_size, 256);
}
#[test]
fn test_ocaml_table_lookup_missing() {
let table = X86GCOCamlTable::new();
assert!(table.lookup(0xDEAD).is_none());
}
#[test]
fn test_coreclr_info_new() {
let info = X86GCCoreCLRInfo::new(0x400000, 1024);
assert_eq!(info.version, 2);
assert!(!info.is_fully_interruptible);
assert_eq!(info.method_size, 1024);
}
#[test]
fn test_coreclr_info_fully_interruptible() {
let mut info = X86GCCoreCLRInfo::new(0x400000, 1024);
info.mark_fully_interruptible();
assert!(info.is_fully_interruptible);
}
#[test]
fn test_coreclr_info_add_region() {
let mut info = X86GCCoreCLRInfo::new(0x400000, 1024);
info.add_region(0x10, 0x50);
assert_eq!(info.interruptible_regions.len(), 1);
assert_eq!(info.interruptible_regions[0].start_offset, 0x10);
assert_eq!(info.interruptible_regions[0].end_offset, 0x50);
}
#[test]
fn test_coreclr_info_add_live_slot() {
let mut info = X86GCCoreCLRInfo::new(0x400000, 1024);
info.add_live_slot(-16, false, true);
assert_eq!(info.live_slots.len(), 1);
assert!(info.live_slots[0].is_pinned);
assert!(!info.live_slots[0].is_byref);
}
#[test]
fn test_coreclr_info_encode() {
let mut info = X86GCCoreCLRInfo::new(0x400000, 512);
info.add_live_slot(-8, false, false);
info.add_transition(0x10, 0x10, true);
let encoded = info.encode();
assert!(!encoded.is_empty());
assert_eq!(encoded[0], 2); }
#[test]
fn test_uleb128_encode() {
let mut buf = Vec::new();
encode_uleb128(&mut buf, 0);
assert_eq!(buf, vec![0]);
buf.clear();
encode_uleb128(&mut buf, 127);
assert_eq!(buf, vec![127]);
buf.clear();
encode_uleb128(&mut buf, 128);
assert_eq!(buf, vec![0x80, 0x01]);
buf.clear();
encode_uleb128(&mut buf, 624485);
assert_eq!(buf.len(), 3);
}
#[test]
fn test_sleb128_encode() {
let mut buf = Vec::new();
encode_sleb128(&mut buf, 0);
assert_eq!(buf, vec![0]);
buf.clear();
encode_sleb128(&mut buf, -1);
assert_eq!(buf, vec![0x7F]);
buf.clear();
encode_sleb128(&mut buf, 64);
assert_eq!(buf, vec![64]);
buf.clear();
encode_sleb128(&mut buf, -64);
assert_eq!(buf, vec![0x40]);
}
#[test]
fn test_config_default() {
let config = X86GCFullConfig::default();
assert!(config.emit_stack_maps);
assert!(config.rewrite_statepoints);
assert!(!config.concurrent_gc);
assert_eq!(
config.collection_algorithm,
X86GCCollectionAlgorithm::MarkSweep
);
}
#[test]
fn test_collection_algorithm_display() {
assert_eq!(format!("{}", X86GCCollectionAlgorithm::Copying), "copying");
assert_eq!(
format!("{}", X86GCCollectionAlgorithm::MarkCompact),
"mark-compact"
);
assert_eq!(format!("{}", X86GCCollectionAlgorithm::None), "none");
}
#[test]
fn test_full_gc_cycle_mark_sweep() {
let mut gc = X86GCFull::new(
X86GCStrategy::Statepoint,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
initial_heap_size: 64 * 1024 * 1024,
..Default::default()
},
);
gc.begin_function("allocate_test", 0x400000);
gc.add_root(X86GCRoot::new_base(1, -8).with_language(X86GCRootLanguage::Rust));
let addr1 = gc.allocate(128);
let addr2 = gc.allocate(256);
assert_ne!(addr1, 0);
assert_ne!(addr2, 0);
assert!(gc.total_allocated() >= 384);
gc.collect();
assert_eq!(gc.gc_cycles(), 1);
gc.end_function();
}
#[test]
fn test_full_gc_cycle_with_finalization() {
let mut gc = X86GCFull::new(
X86GCStrategy::CoreCLR,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
enable_finalization: true,
process_weak_refs: true,
initial_heap_size: 64 * 1024 * 1024,
..Default::default()
},
);
gc.begin_function("test_with_finalizers", 0x500000);
gc.add_root(X86GCRoot::new_base(1, -8));
gc.finalization.register_finalizer(0x1000, 0x2000, 0);
gc.finalization.register_finalizer(0x3000, 0x2000, 1);
gc.finalization
.register_weak_reference(0x7000, 0x8000, true);
gc.allocate(128);
gc.collect();
assert_eq!(gc.gc_cycles(), 1);
assert!(gc.is_enabled());
gc.end_function();
}
#[test]
fn test_full_gc_cycle_with_safepoints() {
let mut gc = X86GCFull::new(
X86GCStrategy::CoreCLR,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::MarkSweep,
insert_loop_polls: true,
initial_heap_size: 64 * 1024 * 1024,
..Default::default()
},
);
gc.begin_function("test_safepoints", 0x600000);
gc.add_safepoint("test_safepoints", 0x20, X86GCSafepointKind::LoopBackedge);
gc.add_safepoint("test_safepoints", 0x40, X86GCSafepointKind::LoopBackedge);
gc.add_safepoint("test_safepoints", 0x60, X86GCSafepointKind::AfterCall);
let fn_safepoints = gc.safepoints.get_function_safepoints("test_safepoints");
assert!(fn_safepoints.is_some());
assert!(fn_safepoints.unwrap().len() >= 3);
gc.end_function();
}
#[test]
fn test_full_gc_cycle_incremental() {
let mut gc = X86GCFull::new(
X86GCStrategy::Statepoint,
X86GCFullConfig {
collection_algorithm: X86GCCollectionAlgorithm::Incremental,
concurrent_gc: true,
initial_heap_size: 64 * 1024 * 1024,
..Default::default()
},
);
gc.begin_function("test_incremental", 0x700000);
gc.add_root(X86GCRoot::new_base(1, -8));
gc.allocate(64);
match gc.incremental_collector {
Some(ref mut inc) => {
inc.start_mark(&[0x10000]);
assert!(inc.marking);
}
None => panic!("Expected incremental collector"),
}
let complete = gc.incremental_marking_step(10);
assert!(complete);
let sweep_done = gc.incremental_sweeping_step();
assert!(sweep_done);
gc.end_function();
}
#[test]
fn test_full_gc_cycle_stack_map_emission() {
let mut gc = X86GCFull::new(X86GCStrategy::Statepoint, X86GCFullConfig::default());
for i in 0..3 {
gc.begin_function(&format!("func_{}", i), 0x400000 + i * 0x1000);
gc.stack_map_generator.live_var_state.record_register(
&format!("ptr_{}", i),
DWARF_REG_RBX,
8,
);
gc.stack_map_generator
.live_var_state
.mark_gc_pointer(&format!("ptr_{}", i));
gc.stack_map_generator.generate_record(true);
gc.end_function();
}
let sm = gc.finalize_stack_maps();
assert_eq!(sm.functions.len(), 3);
assert_eq!(sm.records.len(), 3);
let bytes = sm.emit_bytes();
assert!(bytes.len() > 24 * 3);
let mut parser = X86GCStackMapParser::new();
assert!(parser.parse(&bytes));
assert_eq!(parser.record_count(), 3);
}
}