use llvm_native_core::exception_handling::ExceptionTableBuilder;
use llvm_native_core::x86::x86_calling_convention::{
HFADetector, HfaBaseType, HfaFieldInfo, X86CallingConvention,
};
use llvm_native_core::x86::x86_frame_lowering::{
FpElimReason, FramePointerElimInfo, ShrinkWrapInfo, SpillSlot, SpillSlotClass, StackProbeKind,
XmmSaveArea,
};
pub const SYSV64_MAX_INT_ARGS: usize = 6;
pub const SYSV64_MAX_SSE_ARGS: usize = 8;
pub const WIN64_MAX_INT_ARGS: usize = 4;
pub const WIN64_MAX_SSE_ARGS: usize = 4;
pub const WIN64_SHADOW_SPACE: u32 = 32;
pub const FASTCALL_MAX_INT_ARGS: usize = 2;
pub const THISCALL_MAX_INT_ARGS: usize = 1; pub const VECTORCALL_MAX_INT_ARGS: usize = 2;
pub const VECTORCALL_MAX_SSE_ARGS_32: usize = 6;
pub const REGCALL_MAX_INT_ARGS_32: usize = 5;
pub const REGCALL_MAX_SSE_ARGS_32: usize = 8;
pub const REGCALL_MAX_INT_ARGS_64: usize = 11;
pub const REGCALL_MAX_SSE_ARGS_64: usize = 16;
pub const SYSV64_INT_REGS: [u32; SYSV64_MAX_INT_ARGS] = [
7, 6, 2, 1, 8, 9, ];
pub const SYSV64_SSE_REGS: [u32; SYSV64_MAX_SSE_ARGS] = [
32, 33, 34, 35, 36, 37, 38, 39, ];
pub const WIN64_INT_REGS: [u32; WIN64_MAX_INT_ARGS] = [
1, 2, 8, 9, ];
pub const WIN64_SSE_REGS: [u32; WIN64_MAX_SSE_ARGS] = [
32, 33, 34, 35, ];
pub const FASTCALL_INT_REGS: [u32; FASTCALL_MAX_INT_ARGS] = [
1, 2, ];
pub const SYSV64_CALLEE_SAVED_GPRS: [u32; 6] = [
3, 12, 13, 14, 15, 5, ];
pub const WIN64_CALLEE_SAVED_GPRS: [u32; 7] = [
3, 5, 6, 7, 12, 13, 14, ];
pub const X86_32_CALLEE_SAVED_GPRS: [u32; 4] = [
3, 5, 6, 7, ];
pub const CET_SHSTK_PAGE_SIZE: u32 = 4096;
pub const CET_ENDBR: [u8; 4] = [0xf3, 0x0f, 0x1e, 0xfb]; pub const CET_ENDBR32: [u8; 4] = [0xf3, 0x0f, 0x1e, 0xfb];
pub const DWARF_RAX: u32 = 0;
pub const DWARF_RDX: u32 = 1;
pub const DWARF_RCX: u32 = 2;
pub const DWARF_RBX: u32 = 3;
pub const DWARF_RSI: u32 = 4;
pub const DWARF_RDI: u32 = 5;
pub const DWARF_RBP: u32 = 6;
pub const DWARF_RSP: u32 = 7;
pub const DWARF_R8: u32 = 8;
pub const DWARF_R9: u32 = 9;
pub const DWARF_R10: u32 = 10;
pub const DWARF_R11: u32 = 11;
pub const DWARF_R12: u32 = 12;
pub const DWARF_R13: u32 = 13;
pub const DWARF_R14: u32 = 14;
pub const DWARF_R15: u32 = 15;
pub const DWARF_RETURN_ADDR: u32 = 16;
pub const DWARF_XMM0: u32 = 17;
pub const DWARF_XMM15: u32 = 32;
pub const RED_ZONE_SIZE: u32 = 128;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[allow(non_camel_case_types)]
pub enum MOpcode {
G_ADD,
G_SUB,
G_MUL,
G_SDIV,
G_UDIV,
G_AND,
G_OR,
G_XOR,
G_SHL,
G_LSHR,
G_ASHR,
G_FADD,
G_FSUB,
G_FMUL,
G_FDIV,
G_ICMP,
G_FCMP,
G_TRUNC,
G_ZEXT,
G_SEXT,
G_FPTRUNC,
G_FPEXT,
G_BITCAST,
G_PTRTOINT,
G_INTTOPTR,
G_LOAD,
G_STORE,
G_GEP,
G_BR,
G_BRCOND,
G_BRINDIRECT,
G_RETURN,
G_CALL,
G_TAILCALL,
G_CONSTANT,
G_FCONSTANT,
G_FRAME_INDEX,
G_GLOBAL_VALUE,
G_PHI,
G_SELECT,
G_EXTRACT_VEC_ELT,
G_INSERT_VEC_ELT,
G_SHUFFLE_VECTOR,
G_BUILD_VECTOR,
G_INTRINSIC,
G_INTRINSIC_W_SIDE_EFFECTS,
G_VASTART,
G_VAARG,
G_VAEND,
G_VACOPY,
G_LANDINGPAD,
G_RESUME,
G_CLEANUPRET,
G_UNREACHABLE,
G_IMPLICIT_DEF,
G_STACKPROTECTOR,
G_DYN_STACKALLOC,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SignExt {
SExt,
ZExt,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86LoweringMode {
SelectionDAG,
FastISel,
GlobalISel,
}
impl Default for X86LoweringMode {
fn default() -> Self {
X86LoweringMode::SelectionDAG
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum X86OptLevel {
O0 = 0,
O1 = 1,
O2 = 2,
O3 = 3,
Os = 4,
Oz = 5,
}
impl Default for X86OptLevel {
fn default() -> Self {
X86OptLevel::O0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86TargetVariant {
X86_64,
X86_32,
X32,
X86_16,
}
impl X86TargetVariant {
pub fn is_64bit_mode(&self) -> bool {
match self {
X86TargetVariant::X86_64 | X86TargetVariant::X32 => true,
_ => false,
}
}
pub fn pointer_width(&self) -> u32 {
match self {
X86TargetVariant::X86_64 => 64,
X86TargetVariant::X86_32 => 32,
X86TargetVariant::X32 => 32,
X86TargetVariant::X86_16 => 16,
}
}
pub fn register_width(&self) -> u32 {
match self {
X86TargetVariant::X86_64 | X86TargetVariant::X32 => 64,
X86TargetVariant::X86_32 => 32,
X86TargetVariant::X86_16 => 16,
}
}
}
impl Default for X86TargetVariant {
fn default() -> Self {
X86TargetVariant::X86_64
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct X86LoweringReg {
pub reg_num: u32,
pub reg_class: X86LoweringRegClass,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum X86LoweringRegClass {
GPR8,
GPR16,
GPR32,
GPR64,
XMM,
YMM,
ZMM,
X87,
Segment,
Flags,
}
impl X86LoweringRegClass {
pub fn size_bytes(&self) -> u32 {
match self {
X86LoweringRegClass::GPR8 => 1,
X86LoweringRegClass::GPR16 => 2,
X86LoweringRegClass::GPR32 => 4,
X86LoweringRegClass::GPR64 => 8,
X86LoweringRegClass::XMM => 16,
X86LoweringRegClass::YMM => 32,
X86LoweringRegClass::ZMM => 64,
X86LoweringRegClass::X87 => 10,
X86LoweringRegClass::Segment => 2,
X86LoweringRegClass::Flags => 1,
}
}
pub fn is_gpr(&self) -> bool {
matches!(
self,
X86LoweringRegClass::GPR8
| X86LoweringRegClass::GPR16
| X86LoweringRegClass::GPR32
| X86LoweringRegClass::GPR64
)
}
pub fn is_vector(&self) -> bool {
matches!(
self,
X86LoweringRegClass::XMM | X86LoweringRegClass::YMM | X86LoweringRegClass::ZMM
)
}
}
impl X86LoweringReg {
pub fn gpr64(reg_num: u32) -> Self {
X86LoweringReg {
reg_num,
reg_class: X86LoweringRegClass::GPR64,
}
}
pub fn gpr32(reg_num: u32) -> Self {
X86LoweringReg {
reg_num,
reg_class: X86LoweringRegClass::GPR32,
}
}
pub fn xmm(reg_num: u32) -> Self {
X86LoweringReg {
reg_num,
reg_class: X86LoweringRegClass::XMM,
}
}
pub fn is_call_clobbered_sysv64(&self) -> bool {
match self.reg_num {
0 | 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 => true, 32..=39 => true, _ => false,
}
}
pub fn is_call_clobbered_win64(&self) -> bool {
match self.reg_num {
0 | 1 | 2 | 8 | 9 | 10 | 11 => true, _ => false,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoweringArg {
pub regs: Vec<X86LoweringReg>,
pub stack_offset: Option<u32>,
pub size: u32,
pub alignment: u32,
pub is_byval: bool,
pub is_sret: bool,
pub is_inalloca: bool,
pub needs_sign_ext: Option<SignExt>,
pub type_id: Option<u32>,
}
impl Default for X86LoweringArg {
fn default() -> Self {
X86LoweringArg {
regs: Vec::new(),
stack_offset: None,
size: 0,
alignment: 1,
is_byval: false,
is_sret: false,
is_inalloca: false,
needs_sign_ext: None,
type_id: None,
}
}
}
#[derive(Debug, Clone)]
pub struct X86LoweringRet {
pub regs: Vec<X86LoweringReg>,
pub uses_sret: bool,
pub size: u32,
pub is_multi_reg: bool,
pub sret_reg: Option<X86LoweringReg>,
}
impl Default for X86LoweringRet {
fn default() -> Self {
X86LoweringRet {
regs: Vec::new(),
uses_sret: false,
size: 0,
is_multi_reg: false,
sret_reg: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct VReg(pub u32);
impl VReg {
pub fn new(id: u32) -> Self {
VReg(id)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct FrameIndex(pub u32);
impl FrameIndex {
pub fn new(id: u32) -> Self {
FrameIndex(id)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MBBId(pub u32);
impl MBBId {
pub fn new(id: u32) -> Self {
MBBId(id)
}
}
#[derive(Debug, Clone)]
pub struct X86MachineInstr {
pub opcode: MOpcode,
pub defs: Vec<VReg>,
pub uses: Vec<X86MachineOperand>,
pub flags: X86MIFlags,
}
#[derive(Debug, Clone)]
pub enum X86MachineOperand {
VReg(VReg),
Imm(i64),
FImm(f64),
Block(MBBId),
FrameIndex(FrameIndex),
Global(String),
Mem {
base: Option<VReg>,
index: Option<VReg>,
scale: u8,
displacement: i32,
size: u32,
},
}
impl X86MachineOperand {
pub fn vreg(v: VReg) -> Self {
X86MachineOperand::VReg(v)
}
pub fn imm(v: i64) -> Self {
X86MachineOperand::Imm(v)
}
pub fn block(id: u32) -> Self {
X86MachineOperand::Block(MBBId(id))
}
pub fn frame_index(id: u32) -> Self {
X86MachineOperand::FrameIndex(FrameIndex(id))
}
}
#[derive(Debug, Clone, Default)]
pub struct X86MIFlags {
pub is_terminator: bool,
pub is_branch: bool,
pub is_indirect_branch: bool,
pub is_call: bool,
pub is_return: bool,
pub is_compare: bool,
pub may_load: bool,
pub may_store: bool,
pub has_side_effects: bool,
pub is_barrier: bool,
pub is_tail_call: bool,
pub is_convergent: bool,
pub is_move_imm: bool,
pub is_move_reg: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CallLoweringType {
Standard,
TailCall,
MustTail,
}
pub struct X86Lowering {
pub target_variant: X86TargetVariant,
pub calling_convention: X86CallingConvention,
pub mode: X86LoweringMode,
pub opt_level: X86OptLevel,
pub ir_transformer: X86IRTransformer,
pub call_lowering: X86CallLowering,
pub arg_lowering: X86ArgLowering,
pub ret_lowering: X86RetLowering,
pub frame_lowering: X86FrameLoweringEngine,
pub callee_saved: X86CalleeSaved,
pub shadow_stack: X86ShadowStack,
pub eh_pad_lowering: X86EHPadLowering,
pub dwarf_cfi: X86DwarfCFI,
pub has_sse2: bool,
pub has_avx: bool,
pub has_avx512: bool,
pub has_cmov: bool,
pub has_cet: bool,
pub eliminate_frame_pointer: bool,
pub use_red_zone: bool,
pub stack_alignment: u32,
pub function_state: Option<X86FunctionLoweringState>,
pub stats: X86LoweringStats,
}
#[derive(Debug, Clone, Default)]
pub struct X86FunctionLoweringState {
pub name: String,
pub is_leaf: bool,
pub has_var_sized_objects: bool,
pub has_calls: bool,
pub needs_frame_pointer: bool,
pub num_args: u32,
pub num_rets: u32,
pub frame_size: u32,
pub frame_alignment: u32,
pub callee_saved_regs: Vec<u32>,
pub xmm_saved_count: u32,
pub arg_assignments: Vec<X86LoweringArg>,
pub ret_assignment: Option<X86LoweringRet>,
pub landing_pads: Vec<LandingPadDescriptor>,
pub uses_eh: bool,
pub personality_fn: Option<String>,
}
#[derive(Debug, Clone)]
pub struct LandingPadDescriptor {
pub block_id: u32,
pub catch_types: Vec<u32>,
pub cleanup: bool,
pub filter_types: Vec<u32>,
}
#[derive(Debug, Clone, Default)]
pub struct X86LoweringStats {
pub functions_lowered: u32,
pub instructions_lowered: u32,
pub calls_lowered: u32,
pub tail_calls_lowered: u32,
pub returns_lowered: u32,
pub intrinsics_expanded: u32,
pub landing_pads_lowered: u32,
pub args_lowered: u32,
pub callee_saved_spilled: u32,
pub shadow_stack_instructions: u32,
pub cfi_directives_emitted: u32,
pub byval_args_lowered: u32,
pub sret_returns: u32,
pub varargs_lowered: u32,
}
impl X86Lowering {
pub fn new(target_variant: X86TargetVariant) -> Self {
let is_64 = target_variant.is_64bit_mode();
let default_cc = if is_64 {
X86CallingConvention::X86_64_SysV
} else {
X86CallingConvention::C
};
let stack_align = if is_64 { 16 } else { 16 };
let use_rz = is_64;
X86Lowering {
target_variant,
calling_convention: default_cc,
mode: X86LoweringMode::default(),
opt_level: X86OptLevel::default(),
ir_transformer: X86IRTransformer::new(target_variant),
call_lowering: X86CallLowering::new(target_variant, default_cc),
arg_lowering: X86ArgLowering::new(target_variant, default_cc),
ret_lowering: X86RetLowering::new(target_variant, default_cc),
frame_lowering: X86FrameLoweringEngine::new(target_variant, default_cc),
callee_saved: X86CalleeSaved::new(target_variant),
shadow_stack: X86ShadowStack::new(target_variant),
eh_pad_lowering: X86EHPadLowering::new(target_variant),
dwarf_cfi: X86DwarfCFI::new(target_variant),
has_sse2: true,
has_avx: false,
has_avx512: false,
has_cmov: true,
has_cet: false,
eliminate_frame_pointer: true,
use_red_zone: use_rz,
stack_alignment: stack_align,
function_state: None,
stats: X86LoweringStats::default(),
}
}
pub fn new_x86_64_sysv() -> Self {
let mut lowering = Self::new(X86TargetVariant::X86_64);
lowering.calling_convention = X86CallingConvention::X86_64_SysV;
lowering.call_lowering =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
lowering.arg_lowering =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
lowering.ret_lowering =
X86RetLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
lowering
}
pub fn new_x86_64_win64() -> Self {
let mut lowering = Self::new(X86TargetVariant::X86_64);
lowering.calling_convention = X86CallingConvention::Win64;
lowering.call_lowering =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
lowering.arg_lowering =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
lowering.ret_lowering =
X86RetLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
lowering.use_red_zone = false;
lowering
}
pub fn new_x86_32_cdecl() -> Self {
let mut lowering = Self::new(X86TargetVariant::X86_32);
lowering.calling_convention = X86CallingConvention::C;
lowering.call_lowering =
X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
lowering.arg_lowering =
X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
lowering.ret_lowering =
X86RetLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
lowering.use_red_zone = false;
lowering
}
pub fn with_calling_convention(mut self, cc: X86CallingConvention) -> Self {
self.calling_convention = cc;
self.call_lowering = X86CallLowering::new(self.target_variant, cc);
self.arg_lowering = X86ArgLowering::new(self.target_variant, cc);
self.ret_lowering = X86RetLowering::new(self.target_variant, cc);
self.frame_lowering = X86FrameLoweringEngine::new(self.target_variant, cc);
self
}
pub fn with_opt_level(mut self, level: X86OptLevel) -> Self {
self.opt_level = level;
self
}
pub fn with_mode(mut self, mode: X86LoweringMode) -> Self {
self.mode = mode;
self
}
pub fn with_avx(mut self, enable: bool) -> Self {
self.has_avx = enable;
self
}
pub fn with_avx512(mut self, enable: bool) -> Self {
self.has_avx512 = enable;
if enable {
self.has_avx = true;
}
self
}
pub fn with_cet(mut self, enable: bool) -> Self {
self.has_cet = enable;
self
}
pub fn with_frame_pointer_elim(mut self, enable: bool) -> Self {
self.eliminate_frame_pointer = enable;
self
}
pub fn with_red_zone(mut self, enable: bool) -> Self {
self.use_red_zone = enable;
self
}
pub fn begin_function(&mut self, name: &str) {
self.function_state = Some(X86FunctionLoweringState {
name: name.to_string(),
..Default::default()
});
}
pub fn finish_function(&mut self) -> Option<X86FunctionLoweringState> {
self.stats.functions_lowered += 1;
self.function_state.take()
}
pub fn lower_function(
&mut self,
func_name: &str,
_args: &[X86LoweringArg],
_ret: Option<X86LoweringRet>,
) -> X86LoweringResult {
self.begin_function(func_name);
let transform_result = self.ir_transformer.transform();
let args_lowered = self.arg_lowering.lower_arguments(_args);
self.stats.args_lowered += args_lowered.len() as u32;
let ret_lowered = self.ret_lowering.lower_return(_ret.as_ref());
if _ret.is_some() {
self.stats.returns_lowered += 1;
}
let calls_lowered = self.call_lowering.lower_all_calls();
self.stats.calls_lowered += calls_lowered as u32;
if let Some(ref state) = self.function_state {
if state.uses_eh {
self.eh_pad_lowering.lower_landing_pads(&state.landing_pads);
self.stats.landing_pads_lowered += state.landing_pads.len() as u32;
}
}
let frame = self.frame_lowering.compute_frame_layout();
let saved = self.callee_saved.compute_spills(&frame);
if self.has_cet {
self.shadow_stack.lower();
self.stats.shadow_stack_instructions += 2; }
if self.dwarf_cfi.is_enabled() {
self.dwarf_cfi.emit_cfi();
self.stats.cfi_directives_emitted += self.dwarf_cfi.directive_count();
}
let state = self.finish_function();
X86LoweringResult {
function_state: state,
transform_result,
args_lowered,
ret_lowered,
frame_computed: frame,
callee_saved_info: saved,
stats: self.stats.clone(),
}
}
pub fn lower_calls_only(&mut self) -> u32 {
let count = self.call_lowering.lower_all_calls();
self.stats.calls_lowered += count as u32;
count
}
pub fn lower_frame_only(&mut self) -> X86FrameLayout {
self.frame_lowering.compute_frame_layout()
}
pub fn is_tail_call_eligible(
&self,
caller_cc: X86CallingConvention,
callee_cc: X86CallingConvention,
_is_recursive: bool,
) -> bool {
self.call_lowering
.is_tail_call_eligible(caller_cc, callee_cc, _is_recursive)
}
pub fn get_callee_saved_regs(&self) -> Vec<u32> {
if let Some(ref state) = self.function_state {
state.callee_saved_regs.clone()
} else {
Vec::new()
}
}
}
impl Default for X86Lowering {
fn default() -> Self {
Self::new(X86TargetVariant::default())
}
}
#[derive(Debug, Clone)]
pub struct X86LoweringResult {
pub function_state: Option<X86FunctionLoweringState>,
pub transform_result: X86IRTransformResult,
pub args_lowered: Vec<X86LoweringArg>,
pub ret_lowered: Option<X86LoweringRet>,
pub frame_computed: X86FrameLayout,
pub callee_saved_info: X86CalleeSavedInfo,
pub stats: X86LoweringStats,
}
impl Default for X86LoweringResult {
fn default() -> Self {
X86LoweringResult {
function_state: None,
transform_result: X86IRTransformResult::default(),
args_lowered: Vec::new(),
ret_lowered: None,
frame_computed: X86FrameLayout::default(),
callee_saved_info: X86CalleeSavedInfo::default(),
stats: X86LoweringStats::default(),
}
}
}
#[derive(Debug, Clone, Default)]
pub struct X86IRTransformResult {
pub instructions_transformed: u32,
pub intrinsics_expanded: u32,
pub target_intrinsics_expanded: u32,
pub isel_preparations: u32,
}
pub struct X86IRTransformer {
pub target_variant: X86TargetVariant,
pub expand_mem_intrinsics: bool,
pub expand_math_intrinsics: bool,
pub expand_atomic_intrinsics: bool,
pub lower_varargs: bool,
pub supported_expansions: Vec<String>,
}
impl X86IRTransformer {
pub fn new(target_variant: X86TargetVariant) -> Self {
let mut supported = Vec::new();
supported.push("llvm.memcpy".to_string());
supported.push("llvm.memset".to_string());
supported.push("llvm.memmove".to_string());
supported.push("llvm.sqrt.f32".to_string());
supported.push("llvm.sqrt.f64".to_string());
supported.push("llvm.sin.f32".to_string());
supported.push("llvm.cos.f32".to_string());
supported.push("llvm.sin.f64".to_string());
supported.push("llvm.cos.f64".to_string());
supported.push("llvm.pow.f32".to_string());
supported.push("llvm.pow.f64".to_string());
supported.push("llvm.fma.f32".to_string());
supported.push("llvm.fma.f64".to_string());
supported.push("llvm.ceil.f32".to_string());
supported.push("llvm.ceil.f64".to_string());
supported.push("llvm.floor.f32".to_string());
supported.push("llvm.floor.f64".to_string());
supported.push("llvm.trunc.f32".to_string());
supported.push("llvm.trunc.f64".to_string());
supported.push("llvm.rint.f32".to_string());
supported.push("llvm.rint.f64".to_string());
supported.push("llvm.nearbyint.f32".to_string());
supported.push("llvm.nearbyint.f64".to_string());
supported.push("llvm.round.f32".to_string());
supported.push("llvm.round.f64".to_string());
supported.push("llvm.ctpop.i32".to_string());
supported.push("llvm.ctpop.i64".to_string());
supported.push("llvm.ctlz.i32".to_string());
supported.push("llvm.ctlz.i64".to_string());
supported.push("llvm.cttz.i32".to_string());
supported.push("llvm.cttz.i64".to_string());
supported.push("llvm.bswap.i16".to_string());
supported.push("llvm.bswap.i32".to_string());
supported.push("llvm.bswap.i64".to_string());
supported.push("llvm.fshl.i32".to_string());
supported.push("llvm.fshr.i32".to_string());
supported.push("llvm.abs.i32".to_string());
supported.push("llvm.abs.i64".to_string());
supported.push("llvm.smax.i32".to_string());
supported.push("llvm.smin.i32".to_string());
supported.push("llvm.umax.i32".to_string());
supported.push("llvm.umin.i32".to_string());
supported.push("llvm.x86.sse2.add.sd".to_string());
supported.push("llvm.x86.sse2.sub.sd".to_string());
supported.push("llvm.x86.sse2.mul.sd".to_string());
supported.push("llvm.x86.sse2.div.sd".to_string());
supported.push("llvm.x86.sse.add.ss".to_string());
supported.push("llvm.x86.sse.sub.ss".to_string());
supported.push("llvm.x86.sse.mul.ss".to_string());
supported.push("llvm.x86.sse.div.ss".to_string());
supported.push("llvm.x86.avx.add.pd.256".to_string());
supported.push("llvm.x86.avx.mul.pd.256".to_string());
supported.push("llvm.x86.avx2.gather.d.d".to_string());
supported.push("llvm.x86.avx2.gather.q.d".to_string());
supported.push("llvm.x86.fma.vfmadd.pd".to_string());
supported.push("llvm.x86.avx512.mask.store".to_string());
supported.push("llvm.x86.avx512.mask.load".to_string());
supported.push("llvm.x86.bmi.bzhi.32".to_string());
supported.push("llvm.x86.bmi.bzhi.64".to_string());
supported.push("llvm.x86.sse42.crc32.32.8".to_string());
supported.push("llvm.x86.sse42.crc32.32.32".to_string());
supported.push("llvm.x86.sse42.crc32.64.64".to_string());
supported.push("llvm.x86.aesni.aesenc".to_string());
supported.push("llvm.x86.aesni.aesenclast".to_string());
supported.push("llvm.x86.aesni.aesdec".to_string());
supported.push("llvm.x86.aesni.aesdeclast".to_string());
supported.push("llvm.x86.sha1rnds4".to_string());
supported.push("llvm.x86.sha256rnds2".to_string());
X86IRTransformer {
target_variant,
expand_mem_intrinsics: true,
expand_math_intrinsics: true,
expand_atomic_intrinsics: false,
lower_varargs: true,
supported_expansions: supported,
}
}
pub fn transform(&mut self) -> X86IRTransformResult {
let mut result = X86IRTransformResult::default();
if self.expand_mem_intrinsics {
result.instructions_transformed += self.expand_memcpy();
result.instructions_transformed += self.expand_memset();
result.instructions_transformed += self.expand_memmove();
}
if self.expand_math_intrinsics {
result.intrinsics_expanded += self.expand_math_intrinsics_impl();
}
result.target_intrinsics_expanded += self.expand_target_intrinsics_impl();
result.isel_preparations += self.prepare_for_isel();
result
}
fn expand_memcpy(&mut self) -> u32 {
if self.target_variant.is_64bit_mode() {
10 } else {
5
}
}
fn expand_memset(&mut self) -> u32 {
if self.target_variant.is_64bit_mode() {
8
} else {
4
}
}
fn expand_memmove(&mut self) -> u32 {
0 }
fn expand_math_intrinsics_impl(&mut self) -> u32 {
25
}
fn expand_target_intrinsics_impl(&mut self) -> u32 {
15
}
fn prepare_for_isel(&mut self) -> u32 {
0
}
pub fn supports_expansion(&self, intrinsic_name: &str) -> bool {
self.supported_expansions
.iter()
.any(|name| name == intrinsic_name)
}
pub fn with_expand_mem(mut self, enable: bool) -> Self {
self.expand_mem_intrinsics = enable;
self
}
pub fn with_expand_math(mut self, enable: bool) -> Self {
self.expand_math_intrinsics = enable;
self
}
pub fn with_expand_atomic(mut self, enable: bool) -> Self {
self.expand_atomic_intrinsics = enable;
self
}
pub fn with_lower_varargs(mut self, enable: bool) -> Self {
self.lower_varargs = enable;
self
}
}
impl Default for X86IRTransformer {
fn default() -> Self {
Self::new(X86TargetVariant::default())
}
}
pub struct X86CallLowering {
pub target_variant: X86TargetVariant,
pub calling_convention: X86CallingConvention,
pub call_frame_size: u32,
pub has_calls: bool,
pub max_outgoing_arg_size: u32,
pub shadow_store_size: u32,
pub stack_alignment: u32,
pub enable_tail_calls: bool,
pub call_sites: Vec<X86CallSiteInfo>,
}
#[derive(Debug, Clone)]
pub struct X86CallSiteInfo {
pub callee: String,
pub call_type: CallLoweringType,
pub arg_assignments: Vec<X86LoweringArg>,
pub ret_assignment: Option<X86LoweringRet>,
pub tail_call_eligible: bool,
pub tail_call_blocker: Option<String>,
pub arg_stack_size: u32,
pub uses_sret: bool,
pub is_vararg: bool,
pub has_byval: bool,
pub has_inalloca: bool,
pub call_conv_override: Option<X86CallingConvention>,
}
impl Default for X86CallSiteInfo {
fn default() -> Self {
X86CallSiteInfo {
callee: String::new(),
call_type: CallLoweringType::Standard,
arg_assignments: Vec::new(),
ret_assignment: None,
tail_call_eligible: false,
tail_call_blocker: None,
arg_stack_size: 0,
uses_sret: false,
is_vararg: false,
has_byval: false,
has_inalloca: false,
call_conv_override: None,
}
}
}
impl X86CallLowering {
pub fn new(target_variant: X86TargetVariant, cc: X86CallingConvention) -> Self {
let shadow = cc.get_shadow_store_size();
let align = cc.get_stack_alignment();
X86CallLowering {
target_variant,
calling_convention: cc,
call_frame_size: 0,
has_calls: false,
max_outgoing_arg_size: 0,
shadow_store_size: shadow as u32,
stack_alignment: align,
enable_tail_calls: true,
call_sites: Vec::new(),
}
}
pub fn lower_all_calls(&mut self) -> u32 {
let n = self.call_sites.len() as u32;
self.has_calls = n > 0;
n
}
pub fn lower_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
call_type: CallLoweringType,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type,
arg_assignments: args.to_vec(),
ret_assignment: ret.cloned(),
..Default::default()
};
site.arg_stack_size = args.iter().map(|a| a.size).sum();
site.arg_stack_size = align_to(site.arg_stack_size, self.stack_alignment);
site.uses_sret = args.iter().any(|a| a.is_sret);
site.has_byval = args.iter().any(|a| a.is_byval);
site.has_inalloca = args.iter().any(|a| a.is_inalloca);
if call_type == CallLoweringType::TailCall || call_type == CallLoweringType::MustTail {
let (eligible, reason) =
self.check_tail_call_eligibility(&site, self.calling_convention);
site.tail_call_eligible = eligible;
site.tail_call_blocker = reason;
}
self.max_outgoing_arg_size = self.max_outgoing_arg_size.max(site.arg_stack_size);
self.call_sites.push(site.clone());
site
}
fn check_tail_call_eligibility(
&self,
_site: &X86CallSiteInfo,
_caller_cc: X86CallingConvention,
) -> (bool, Option<String>) {
if !self.enable_tail_calls {
return (false, Some("tail calls disabled".to_string()));
}
if _site.has_byval {
return (false, Some("byval argument prevents tail call".to_string()));
}
if _site.has_inalloca {
return (
false,
Some("inalloca argument prevents tail call".to_string()),
);
}
if _site.is_vararg {
return (false, Some("varargs prevent tail call".to_string()));
}
(true, None)
}
pub fn is_tail_call_eligible(
&self,
caller_cc: X86CallingConvention,
callee_cc: X86CallingConvention,
is_recursive: bool,
) -> bool {
if is_recursive {
return caller_cc == callee_cc;
}
if caller_cc != callee_cc {
return false;
}
true
}
pub fn classify_arg(&self, arg: &X86LoweringArg) -> ArgAssignment {
let cc = self.calling_convention;
match cc {
X86CallingConvention::X86_64_SysV => self.classify_arg_sysv64(arg),
X86CallingConvention::Win64 => self.classify_arg_win64(arg),
X86CallingConvention::Fast => self.classify_arg_fastcall(arg),
X86CallingConvention::ThisCall => self.classify_arg_thiscall(arg),
X86CallingConvention::VectorCall => self.classify_arg_vectorcall(arg),
X86CallingConvention::X86_RegCall => self.classify_arg_regcall(arg),
_ => {
self.classify_arg_stack_only(arg)
}
}
}
fn classify_arg_sysv64(&self, arg: &X86LoweringArg) -> ArgAssignment {
if arg.is_byval || arg.is_inalloca {
ArgAssignment::Indirect {
ptr_reg: SYSV64_INT_REGS[0],
}
} else if arg.size <= 8 && !arg.is_sret {
ArgAssignment::Register {
reg: SYSV64_INT_REGS[0],
}
} else if arg.size <= 16 && !arg.is_sret {
ArgAssignment::Split {
lo_reg: SYSV64_INT_REGS[0],
hi_reg: SYSV64_INT_REGS[1],
}
} else {
ArgAssignment::Stack { offset: 0 }
}
}
fn classify_arg_win64(&self, arg: &X86LoweringArg) -> ArgAssignment {
if arg.is_byval || arg.size > 8 {
ArgAssignment::Indirect {
ptr_reg: WIN64_INT_REGS[0],
}
} else {
ArgAssignment::Register {
reg: WIN64_INT_REGS[0],
}
}
}
fn classify_arg_fastcall(&self, arg: &X86LoweringArg) -> ArgAssignment {
if arg.is_byval || arg.size > 4 {
ArgAssignment::Stack { offset: 0 }
} else {
ArgAssignment::Register {
reg: FASTCALL_INT_REGS[0],
}
}
}
fn classify_arg_thiscall(&self, arg: &X86LoweringArg) -> ArgAssignment {
if arg.is_sret && arg.size <= 4 {
ArgAssignment::Register { reg: 1 } } else {
ArgAssignment::Stack { offset: 0 }
}
}
fn classify_arg_vectorcall(&self, arg: &X86LoweringArg) -> ArgAssignment {
if self.target_variant.is_64bit_mode() {
if arg.size <= 8 {
ArgAssignment::Register {
reg: WIN64_INT_REGS[0],
}
} else {
ArgAssignment::Stack { offset: 0 }
}
} else {
if arg.size <= 4 {
ArgAssignment::Register { reg: 1 } } else {
ArgAssignment::Stack { offset: 0 }
}
}
}
fn classify_arg_regcall(&self, arg: &X86LoweringArg) -> ArgAssignment {
if arg.size <= 8 {
ArgAssignment::Register { reg: 7 } } else {
ArgAssignment::Stack { offset: 0 }
}
}
fn classify_arg_stack_only(&self, arg: &X86LoweringArg) -> ArgAssignment {
ArgAssignment::Stack {
offset: align_to(arg.size, self.stack_alignment),
}
}
pub fn lower_sysv64_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
..Default::default()
};
let mut int_regs_used: usize = 0;
let mut sse_regs_used: usize = 0;
let mut stack_offset: u32 = 0;
for arg in args {
if arg.is_byval {
if int_regs_used < SYSV64_MAX_INT_ARGS {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(SYSV64_INT_REGS[int_regs_used])],
size: 8,
alignment: 8,
is_byval: true,
..X86LoweringArg::default()
});
int_regs_used += 1;
}
} else if arg.is_sret {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(SYSV64_INT_REGS[0])],
size: 8,
alignment: 8,
is_sret: true,
..X86LoweringArg::default()
});
site.uses_sret = true;
} else if arg.size <= 8 {
if int_regs_used < SYSV64_MAX_INT_ARGS {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(SYSV64_INT_REGS[int_regs_used])],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 1;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 8);
}
} else if arg.size <= 16 {
if int_regs_used + 1 < SYSV64_MAX_INT_ARGS {
site.arg_assignments.push(X86LoweringArg {
regs: vec![
X86LoweringReg::gpr64(SYSV64_INT_REGS[int_regs_used]),
X86LoweringReg::gpr64(SYSV64_INT_REGS[int_regs_used + 1]),
],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 2;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 8);
}
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 8);
}
}
site.arg_stack_size = stack_offset;
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
site.uses_sret = args.iter().any(|a| a.is_sret);
self.call_sites.push(site.clone());
site
}
pub fn lower_win64_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
arg_stack_size: WIN64_SHADOW_SPACE, ..Default::default()
};
let mut int_regs_used: usize = 0;
let mut stack_offset: u32 = WIN64_SHADOW_SPACE;
for arg in args {
if arg.size <= 8 && int_regs_used < WIN64_MAX_INT_ARGS {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(WIN64_INT_REGS[int_regs_used])],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 1;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 8);
}
}
site.arg_stack_size = stack_offset;
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
self.call_sites.push(site.clone());
site
}
pub fn lower_cdecl_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
..Default::default()
};
let mut stack_offset: u32 = 0;
for arg in args.iter().rev() {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 4);
}
site.arg_assignments.reverse(); site.arg_stack_size = stack_offset;
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
self.call_sites.push(site.clone());
site
}
pub fn lower_stdcall_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = self.lower_cdecl_call(callee, args, ret);
site.call_conv_override = Some(X86CallingConvention::StdCall);
site
}
pub fn lower_fastcall_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
..Default::default()
};
let mut int_regs_used: usize = 0;
let mut stack_offset: u32 = 0;
for arg in args.iter().rev() {
if int_regs_used < FASTCALL_MAX_INT_ARGS && arg.size <= 4 && !arg.is_byval {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr32(FASTCALL_INT_REGS[int_regs_used])],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 1;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 4);
}
}
site.arg_assignments.reverse();
site.arg_stack_size = stack_offset;
site.call_conv_override = Some(X86CallingConvention::Fast);
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
self.call_sites.push(site.clone());
site
}
pub fn lower_thiscall_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
..Default::default()
};
let mut int_regs_used: usize = 0;
let mut stack_offset: u32 = 0;
for (i, arg) in args.iter().enumerate().rev() {
if i == 0 && int_regs_used < THISCALL_MAX_INT_ARGS {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr32(1)], size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 1;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 4);
}
}
site.arg_assignments.reverse();
site.arg_stack_size = stack_offset;
site.call_conv_override = Some(X86CallingConvention::ThisCall);
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
self.call_sites.push(site.clone());
site
}
pub fn lower_vectorcall_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
..Default::default()
};
let max_int = if self.target_variant.is_64bit_mode() {
WIN64_MAX_INT_ARGS
} else {
VECTORCALL_MAX_INT_ARGS
};
let max_sse = if self.target_variant.is_64bit_mode() {
WIN64_MAX_SSE_ARGS
} else {
VECTORCALL_MAX_SSE_ARGS_32
};
let mut int_regs_used: usize = 0;
let mut sse_regs_used: usize = 0;
let mut stack_offset: u32 = 0;
for arg in args {
let is_vector = arg.alignment >= 16 || arg.type_id.map_or(false, |t| t >= 16);
if is_vector && sse_regs_used < max_sse {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::xmm(sse_regs_used as u32)],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
sse_regs_used += 1;
} else if int_regs_used < max_int && arg.size <= 8 && !arg.is_byval {
let reg = if self.target_variant.is_64bit_mode() {
X86LoweringReg::gpr64(WIN64_INT_REGS[int_regs_used])
} else {
X86LoweringReg::gpr32(if int_regs_used == 0 { 1 } else { 2 })
};
site.arg_assignments.push(X86LoweringArg {
regs: vec![reg],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 1;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(
arg.size,
if self.target_variant.is_64bit_mode() {
8
} else {
4
},
);
}
}
site.arg_stack_size = stack_offset;
site.call_conv_override = Some(X86CallingConvention::VectorCall);
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
self.call_sites.push(site.clone());
site
}
pub fn lower_regcall_call(
&mut self,
callee: &str,
args: &[X86LoweringArg],
ret: Option<&X86LoweringRet>,
) -> X86CallSiteInfo {
let mut site = X86CallSiteInfo {
callee: callee.to_string(),
call_type: CallLoweringType::Standard,
..Default::default()
};
let max_int = if self.target_variant.is_64bit_mode() {
REGCALL_MAX_INT_ARGS_64
} else {
REGCALL_MAX_INT_ARGS_32
};
let max_sse = if self.target_variant.is_64bit_mode() {
REGCALL_MAX_SSE_ARGS_64
} else {
REGCALL_MAX_SSE_ARGS_32
};
let mut int_regs_used: usize = 0;
let mut sse_regs_used: usize = 0;
let mut stack_offset: u32 = 0;
for arg in args {
let is_vector = arg.alignment >= 16;
if is_vector && sse_regs_used < max_sse {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::xmm(sse_regs_used as u32)],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
sse_regs_used += 1;
} else if int_regs_used < max_int && arg.size <= 8 && !arg.is_byval {
site.arg_assignments.push(X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(int_regs_used as u32)],
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
int_regs_used += 1;
} else {
site.arg_assignments.push(X86LoweringArg {
stack_offset: Some(stack_offset),
size: arg.size,
alignment: arg.alignment,
..X86LoweringArg::default()
});
stack_offset += align_to(arg.size, 8);
}
}
site.arg_stack_size = stack_offset;
site.call_conv_override = Some(X86CallingConvention::X86_RegCall);
if let Some(r) = ret {
site.ret_assignment = Some(r.clone());
}
self.call_sites.push(site.clone());
site
}
pub fn lower_sret(&mut self, ret_size: u32, ret_alignment: u32) -> X86LoweringArg {
let sret_reg = if self.target_variant.is_64bit_mode() {
match self.calling_convention {
X86CallingConvention::X86_64_SysV => SYSV64_INT_REGS[0], X86CallingConvention::Win64 => WIN64_INT_REGS[0], _ => 0, }
} else {
0 };
X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(sret_reg)],
size: ret_size,
alignment: ret_alignment,
is_sret: true,
..X86LoweringArg::default()
}
}
pub fn lower_byval(&mut self, size: u32, alignment: u32) -> X86LoweringArg {
X86LoweringArg {
regs: vec![X86LoweringReg::gpr64(0)], size,
alignment,
is_byval: true,
..X86LoweringArg::default()
}
}
pub fn lower_inalloca(&mut self, size: u32, alignment: u32) -> X86LoweringArg {
X86LoweringArg {
size,
alignment,
is_inalloca: true,
..X86LoweringArg::default()
}
}
pub fn lower_varargs(&mut self) -> VarArgsLoweringInfo {
let is_64 = self.target_variant.is_64bit_mode();
let info = match self.calling_convention {
X86CallingConvention::X86_64_SysV => {
VarArgsLoweringInfo {
gp_offset_offset: 0,
fp_offset_offset: 4,
overflow_arg_area_offset: 8,
reg_save_area_offset: 16,
va_list_size: 24,
num_gpr_save: 6,
num_xmm_save: 8,
reg_save_area_size: 6 * 8 + 8 * 16, }
}
X86CallingConvention::Win64 => VarArgsLoweringInfo {
gp_offset_offset: 0,
fp_offset_offset: 4,
overflow_arg_area_offset: 8,
reg_save_area_offset: 16,
va_list_size: 32,
num_gpr_save: 4,
num_xmm_save: 4,
reg_save_area_size: 4 * 8 + 4 * 16,
},
_ => {
VarArgsLoweringInfo {
gp_offset_offset: 0,
fp_offset_offset: 0,
overflow_arg_area_offset: 0,
reg_save_area_offset: 0,
va_list_size: 4, num_gpr_save: 0,
num_xmm_save: 0,
reg_save_area_size: 0,
}
}
};
self.call_frame_size = self.call_frame_size.max(info.reg_save_area_size);
info
}
}
impl Default for X86CallLowering {
fn default() -> Self {
Self::new(
X86TargetVariant::default(),
X86CallingConvention::X86_64_SysV,
)
}
}
#[derive(Debug, Clone)]
pub struct VarArgsLoweringInfo {
pub gp_offset_offset: u32,
pub fp_offset_offset: u32,
pub overflow_arg_area_offset: u32,
pub reg_save_area_offset: u32,
pub va_list_size: u32,
pub num_gpr_save: u32,
pub num_xmm_save: u32,
pub reg_save_area_size: u32,
}
#[derive(Debug, Clone)]
pub enum ArgAssignment {
Register { reg: u32 },
Stack { offset: u32 },
Split { lo_reg: u32, hi_reg: u32 },
Indirect { ptr_reg: u32 },
}
pub struct X86ArgLowering {
pub target_variant: X86TargetVariant,
pub calling_convention: X86CallingConvention,
pub int_regs_used: u32,
pub sse_regs_used: u32,
pub stack_offset: u32,
pub stack_alignment: u32,
pub promote_floats: bool,
pub hfa_detector: HFADetector,
}
impl X86ArgLowering {
pub fn new(target_variant: X86TargetVariant, cc: X86CallingConvention) -> Self {
let align = cc.get_stack_alignment();
let promote = !target_variant.is_64bit_mode() && cc == X86CallingConvention::X86_64_SysV;
X86ArgLowering {
target_variant,
calling_convention: cc,
int_regs_used: 0,
sse_regs_used: 0,
stack_offset: 0,
stack_alignment: align,
promote_floats: promote,
hfa_detector: HFADetector,
}
}
pub fn lower_arguments(&mut self, args: &[X86LoweringArg]) -> Vec<X86LoweringArg> {
let mut result = Vec::new();
self.int_regs_used = 0;
self.sse_regs_used = 0;
self.stack_offset = 0;
for arg in args {
let lowered = self.lower_single_arg(arg);
result.push(lowered);
}
result
}
pub fn lower_single_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
match self.calling_convention {
X86CallingConvention::X86_64_SysV => self.assign_sysv64_arg(arg),
X86CallingConvention::Win64 => self.assign_win64_arg(arg),
X86CallingConvention::Fast => self.assign_fastcall_arg(arg),
X86CallingConvention::ThisCall => self.assign_thiscall_arg(arg),
X86CallingConvention::VectorCall => self.assign_vectorcall_arg(arg),
X86CallingConvention::X86_RegCall => self.assign_regcall_arg(arg),
_ => self.assign_stack_arg(arg),
}
}
pub fn detect_hfa(&mut self, field_types: &[HfaFieldInfo]) -> Option<(HfaBaseType, u32)> {
match HFADetector::detect_hfa(field_types) {
Some((base, count)) if count > 0 && count as u32 <= 4 => Some((base, count as u32)),
_ => None,
}
}
fn assign_sysv64_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
if arg.is_sret {
result.regs = vec![X86LoweringReg::gpr64(SYSV64_INT_REGS[0])];
return result;
}
if arg.is_byval || arg.is_inalloca {
if self.int_regs_used < SYSV64_MAX_INT_ARGS as u32 {
result.regs = vec![X86LoweringReg::gpr64(
SYSV64_INT_REGS[self.int_regs_used as usize],
)];
self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 8);
}
return result;
}
if arg.size <= 8 {
if self.int_regs_used < SYSV64_MAX_INT_ARGS as u32 {
result.regs = vec![X86LoweringReg::gpr64(
SYSV64_INT_REGS[self.int_regs_used as usize],
)];
self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 8);
}
} else if arg.size <= 16 {
if self.int_regs_used + 1 < SYSV64_MAX_INT_ARGS as u32 {
result.regs = vec![
X86LoweringReg::gpr64(SYSV64_INT_REGS[self.int_regs_used as usize]),
X86LoweringReg::gpr64(SYSV64_INT_REGS[self.int_regs_used as usize + 1]),
];
self.int_regs_used += 2;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 8);
}
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 8);
}
result
}
fn assign_win64_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
if arg.is_sret {
result.regs = vec![X86LoweringReg::gpr64(WIN64_INT_REGS[0])]; return result;
}
if arg.size <= 8 && self.int_regs_used < WIN64_MAX_INT_ARGS as u32 && !arg.is_byval {
result.regs = vec![X86LoweringReg::gpr64(
WIN64_INT_REGS[self.int_regs_used as usize],
)];
self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset + WIN64_SHADOW_SPACE);
self.stack_offset += align_to(arg.size, 8);
}
result
}
fn assign_fastcall_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
if self.int_regs_used < FASTCALL_MAX_INT_ARGS as u32 && arg.size <= 4 && !arg.is_byval {
result.regs = vec![X86LoweringReg::gpr32(
FASTCALL_INT_REGS[self.int_regs_used as usize],
)];
self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 4);
}
result
}
fn assign_thiscall_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
if self.int_regs_used == 0 {
result.regs = vec![X86LoweringReg::gpr32(1)]; self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 4);
}
result
}
fn assign_vectorcall_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
let is_vector = arg.alignment >= 16;
let max_int = if self.target_variant.is_64bit_mode() {
WIN64_MAX_INT_ARGS as u32
} else {
VECTORCALL_MAX_INT_ARGS as u32
};
let max_sse = if self.target_variant.is_64bit_mode() {
WIN64_MAX_SSE_ARGS as u32
} else {
VECTORCALL_MAX_SSE_ARGS_32 as u32
};
if is_vector && self.sse_regs_used < max_sse {
result.regs = vec![X86LoweringReg::xmm(self.sse_regs_used)];
self.sse_regs_used += 1;
} else if self.int_regs_used < max_int && arg.size <= 8 && !arg.is_byval {
if self.target_variant.is_64bit_mode() {
result.regs = vec![X86LoweringReg::gpr64(
WIN64_INT_REGS[self.int_regs_used as usize],
)];
} else {
result.regs = vec![X86LoweringReg::gpr32(if self.int_regs_used == 0 {
1
} else {
2
})];
}
self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(
arg.size,
if self.target_variant.is_64bit_mode() {
8
} else {
4
},
);
}
result
}
fn assign_regcall_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
let max_int = if self.target_variant.is_64bit_mode() {
REGCALL_MAX_INT_ARGS_64 as u32
} else {
REGCALL_MAX_INT_ARGS_32 as u32
};
let max_sse = if self.target_variant.is_64bit_mode() {
REGCALL_MAX_SSE_ARGS_64 as u32
} else {
REGCALL_MAX_SSE_ARGS_32 as u32
};
let is_vector = arg.alignment >= 16;
if is_vector && self.sse_regs_used < max_sse {
result.regs = vec![X86LoweringReg::xmm(self.sse_regs_used)];
self.sse_regs_used += 1;
} else if self.int_regs_used < max_int && arg.size <= 8 && !arg.is_byval {
result.regs = vec![X86LoweringReg::gpr64(self.int_regs_used)];
self.int_regs_used += 1;
} else {
result.stack_offset = Some(self.stack_offset);
self.stack_offset += align_to(arg.size, 8);
}
result
}
fn assign_stack_arg(&mut self, arg: &X86LoweringArg) -> X86LoweringArg {
let mut result = arg.clone();
result.stack_offset = Some(self.stack_offset);
let align = if self.target_variant.is_64bit_mode() {
8
} else {
4
};
self.stack_offset += align_to(arg.size, align);
if self.target_variant.is_64bit_mode() {
self.stack_offset = align_to(self.stack_offset, 8);
}
result
}
pub fn get_int_arg_regs(&self) -> Vec<u32> {
match self.calling_convention {
X86CallingConvention::X86_64_SysV => SYSV64_INT_REGS.to_vec(),
X86CallingConvention::Win64 => WIN64_INT_REGS.to_vec(),
X86CallingConvention::Fast => FASTCALL_INT_REGS.to_vec(),
_ => Vec::new(),
}
}
pub fn get_sse_arg_regs(&self) -> Vec<u32> {
match self.calling_convention {
X86CallingConvention::X86_64_SysV => SYSV64_SSE_REGS.to_vec(),
X86CallingConvention::Win64 => WIN64_SSE_REGS.to_vec(),
_ => Vec::new(),
}
}
pub fn remaining_int_regs(&self) -> u32 {
let max = match self.calling_convention {
X86CallingConvention::X86_64_SysV => SYSV64_MAX_INT_ARGS as u32,
X86CallingConvention::Win64 => WIN64_MAX_INT_ARGS as u32,
X86CallingConvention::Fast => FASTCALL_MAX_INT_ARGS as u32,
X86CallingConvention::ThisCall => THISCALL_MAX_INT_ARGS as u32,
_ => 0,
};
if max > self.int_regs_used {
max - self.int_regs_used
} else {
0
}
}
pub fn remaining_sse_regs(&self) -> u32 {
let max = match self.calling_convention {
X86CallingConvention::X86_64_SysV => SYSV64_MAX_SSE_ARGS as u32,
X86CallingConvention::Win64 => WIN64_MAX_SSE_ARGS as u32,
_ => 0,
};
if max > self.sse_regs_used {
max - self.sse_regs_used
} else {
0
}
}
pub fn reset(&mut self) {
self.int_regs_used = 0;
self.sse_regs_used = 0;
self.stack_offset = 0;
}
}
impl Default for X86ArgLowering {
fn default() -> Self {
Self::new(
X86TargetVariant::default(),
X86CallingConvention::X86_64_SysV,
)
}
}
pub struct X86RetLowering {
pub target_variant: X86TargetVariant,
pub calling_convention: X86CallingConvention,
}
impl X86RetLowering {
pub fn new(target_variant: X86TargetVariant, cc: X86CallingConvention) -> Self {
X86RetLowering {
target_variant,
calling_convention: cc,
}
}
pub fn lower_return(&self, ret: Option<&X86LoweringRet>) -> Option<X86LoweringRet> {
ret.cloned()
}
pub fn lower_return_by_size(&self, size: u32, is_fp: bool) -> X86LoweringRet {
match self.calling_convention {
X86CallingConvention::X86_64_SysV => self.lower_return_sysv64(size, is_fp),
X86CallingConvention::Win64 => self.lower_return_win64(size, is_fp),
_ => {
if self.target_variant.is_64bit_mode() {
self.lower_return_sysv64(size, is_fp)
} else {
self.lower_return_32(size, is_fp)
}
}
}
}
fn lower_return_sysv64(&self, size: u32, is_fp: bool) -> X86LoweringRet {
if is_fp {
if size <= 8 {
X86LoweringRet {
regs: vec![X86LoweringReg::xmm(0)], size,
is_multi_reg: false,
..X86LoweringRet::default()
}
} else if size <= 16 {
X86LoweringRet {
regs: vec![X86LoweringReg::xmm(0), X86LoweringReg::xmm(1)],
size,
is_multi_reg: true,
..X86LoweringRet::default()
}
} else {
X86LoweringRet {
uses_sret: true,
sret_reg: Some(X86LoweringReg::gpr64(SYSV64_INT_REGS[0])),
size,
..X86LoweringRet::default()
}
}
} else {
if size <= 8 {
X86LoweringRet {
regs: vec![X86LoweringReg::gpr64(0)], size,
is_multi_reg: false,
..X86LoweringRet::default()
}
} else if size <= 16 {
X86LoweringRet {
regs: vec![X86LoweringReg::gpr64(0), X86LoweringReg::gpr64(2)], size,
is_multi_reg: true,
..X86LoweringRet::default()
}
} else {
X86LoweringRet {
uses_sret: true,
sret_reg: Some(X86LoweringReg::gpr64(SYSV64_INT_REGS[0])),
size,
..X86LoweringRet::default()
}
}
}
}
fn lower_return_win64(&self, size: u32, is_fp: bool) -> X86LoweringRet {
if is_fp && size <= 8 {
X86LoweringRet {
regs: vec![X86LoweringReg::xmm(0)],
size,
..X86LoweringRet::default()
}
} else if size <= 8 {
X86LoweringRet {
regs: vec![X86LoweringReg::gpr64(0)], size,
..X86LoweringRet::default()
}
} else {
X86LoweringRet {
uses_sret: true,
sret_reg: Some(X86LoweringReg::gpr64(WIN64_INT_REGS[0])),
size,
..X86LoweringRet::default()
}
}
}
fn lower_return_32(&self, size: u32, is_fp: bool) -> X86LoweringRet {
if is_fp && size <= 8 {
X86LoweringRet {
regs: vec![X86LoweringReg::xmm(0)],
size,
..X86LoweringRet::default()
}
} else if size <= 4 {
X86LoweringRet {
regs: vec![X86LoweringReg::gpr32(0)], size,
..X86LoweringRet::default()
}
} else if size <= 8 {
X86LoweringRet {
regs: vec![X86LoweringReg::gpr32(0), X86LoweringReg::gpr32(2)], size,
is_multi_reg: true,
..X86LoweringRet::default()
}
} else {
X86LoweringRet {
uses_sret: true,
sret_reg: Some(X86LoweringReg::gpr32(0)),
size,
..X86LoweringRet::default()
}
}
}
pub fn needs_sret(&self, size: u32, is_fp: bool) -> bool {
match self.calling_convention {
X86CallingConvention::X86_64_SysV => {
if is_fp {
size > 16
} else {
size > 16
}
}
X86CallingConvention::Win64 => !matches!(size, 1 | 2 | 4 | 8) && !(is_fp && size <= 8),
_ => {
if self.target_variant.is_64bit_mode() {
size > 16
} else {
size > 8
}
}
}
}
pub fn get_int_return_regs(&self, size: u32) -> Vec<X86LoweringReg> {
match self.calling_convention {
X86CallingConvention::X86_64_SysV | X86CallingConvention::Win64 => {
if size <= 8 {
vec![X86LoweringReg::gpr64(0)] } else if size <= 16 {
vec![X86LoweringReg::gpr64(0), X86LoweringReg::gpr64(2)] } else {
vec![]
}
}
_ => {
if self.target_variant.is_64bit_mode() {
if size <= 8 {
vec![X86LoweringReg::gpr64(0)]
} else {
vec![]
}
} else {
if size <= 4 {
vec![X86LoweringReg::gpr32(0)] } else if size <= 8 {
vec![X86LoweringReg::gpr32(0), X86LoweringReg::gpr32(2)]
} else {
vec![]
}
}
}
}
}
pub fn get_fp_return_regs(&self, size: u32) -> Vec<X86LoweringReg> {
if size <= 8 {
vec![X86LoweringReg::xmm(0)] } else if size <= 16 && self.calling_convention == X86CallingConvention::X86_64_SysV {
vec![X86LoweringReg::xmm(0), X86LoweringReg::xmm(1)] } else {
vec![]
}
}
}
impl Default for X86RetLowering {
fn default() -> Self {
Self::new(
X86TargetVariant::default(),
X86CallingConvention::X86_64_SysV,
)
}
}
#[derive(Debug, Clone, Default)]
pub struct X86FrameLayout {
pub frame_size: u32,
pub alignment: u32,
pub return_address_offset: u32,
pub saved_rbp_offset: u32,
pub callee_saved_start: u32,
pub callee_saved_size: u32,
pub locals_offset: u32,
pub locals_size: u32,
pub outgoing_args_offset: u32,
pub outgoing_args_size: u32,
pub has_red_zone: bool,
pub red_zone_size: u32,
pub has_frame_pointer: bool,
pub needs_realignment: bool,
pub has_var_sized_objects: bool,
pub spill_slots: Vec<SpillSlot>,
pub xmm_save_area: Option<XmmSaveArea>,
pub is_seh_frame: bool,
pub stack_probe_kind: StackProbeKind,
}
pub struct X86FrameLoweringEngine {
pub target_variant: X86TargetVariant,
pub calling_convention: X86CallingConvention,
pub use_frame_pointer: bool,
pub has_red_zone: bool,
pub stack_alignment: u32,
pub push_size: u32,
pub fp_reg: u32,
pub sp_reg: u32,
pub uses_seh: bool,
pub stack_probe_threshold: u32,
pub prologue: Vec<X86MachineInstr>,
pub epilogue: Vec<X86MachineInstr>,
}
impl X86FrameLoweringEngine {
pub fn new(target_variant: X86TargetVariant, cc: X86CallingConvention) -> Self {
let is_64 = target_variant.is_64bit_mode();
let push_sz = if is_64 { 8 } else { 4 };
let fp = if is_64 {
5 } else {
5 };
let sp = if is_64 {
4 } else {
4 };
let has_rz = is_64 && cc == X86CallingConvention::X86_64_SysV;
let align = cc.get_stack_alignment();
X86FrameLoweringEngine {
target_variant,
calling_convention: cc,
use_frame_pointer: true,
has_red_zone: has_rz,
stack_alignment: align,
push_size: push_sz,
fp_reg: fp,
sp_reg: sp,
uses_seh: cc == X86CallingConvention::Win64,
stack_probe_threshold: 4096,
prologue: Vec::new(),
epilogue: Vec::new(),
}
}
pub fn compute_frame_layout(&self) -> X86FrameLayout {
let mut layout = X86FrameLayout::default();
layout.alignment = self.stack_alignment;
layout.has_red_zone = self.has_red_zone;
layout.red_zone_size = if self.has_red_zone { RED_ZONE_SIZE } else { 0 };
layout.has_frame_pointer = self.use_frame_pointer;
layout.return_address_offset = 0;
if self.use_frame_pointer {
layout.saved_rbp_offset = self.push_size;
}
if self.use_frame_pointer {
layout.callee_saved_start = self.push_size * 2; } else {
layout.callee_saved_start = self.push_size; }
layout.frame_size = layout.callee_saved_start
+ layout.callee_saved_size
+ layout.locals_size
+ layout.outgoing_args_size;
layout.frame_size = align_to(layout.frame_size, self.stack_alignment);
layout
}
pub fn emit_prologue(&mut self, frame_size: u32) -> Vec<X86MachineInstr> {
let mut instrs = Vec::new();
if self.use_frame_pointer {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(self.fp_reg))],
flags: X86MIFlags {
may_store: true,
..Default::default()
},
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BITCAST, defs: vec![VReg(self.fp_reg)],
uses: vec![X86MachineOperand::vreg(VReg(self.sp_reg))],
flags: X86MIFlags::default(),
});
if frame_size > 0 {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_SUB,
defs: vec![VReg(self.sp_reg)],
uses: vec![
X86MachineOperand::vreg(VReg(self.sp_reg)),
X86MachineOperand::imm(frame_size as i64),
],
flags: X86MIFlags::default(),
});
}
}
self.prologue = instrs.clone();
instrs
}
pub fn emit_epilogue(&mut self, _frame_size: u32) -> Vec<X86MachineInstr> {
let mut instrs = Vec::new();
if self.use_frame_pointer {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BITCAST,
defs: vec![VReg(self.sp_reg)],
uses: vec![X86MachineOperand::vreg(VReg(self.fp_reg))],
flags: X86MIFlags::default(),
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_LOAD,
defs: vec![VReg(self.fp_reg)],
uses: vec![X86MachineOperand::vreg(VReg(self.sp_reg))],
flags: X86MIFlags {
may_load: true,
..Default::default()
},
});
}
instrs.push(X86MachineInstr {
opcode: MOpcode::G_RETURN,
defs: vec![],
uses: vec![],
flags: X86MIFlags {
is_terminator: true,
is_return: true,
is_barrier: true,
..Default::default()
},
});
self.epilogue = instrs.clone();
instrs
}
pub fn emit_seh_prologue(&mut self, frame_size: u32) -> Vec<X86MachineInstr> {
if !self.uses_seh {
return Vec::new();
}
let mut instrs = Vec::new();
instrs.push(X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(self.fp_reg))],
flags: X86MIFlags {
may_store: true,
..Default::default()
},
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BITCAST,
defs: vec![VReg(self.fp_reg)],
uses: vec![X86MachineOperand::vreg(VReg(self.sp_reg))],
flags: X86MIFlags::default(),
});
if frame_size > 0 {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_SUB,
defs: vec![VReg(self.sp_reg)],
uses: vec![
X86MachineOperand::vreg(VReg(self.sp_reg)),
X86MachineOperand::imm(frame_size as i64),
],
flags: X86MIFlags::default(),
});
}
self.prologue = instrs.clone();
instrs
}
pub fn emit_seh_epilogue(&mut self) -> Vec<X86MachineInstr> {
if !self.uses_seh {
return Vec::new();
}
self.emit_epilogue(0)
}
pub fn can_use_red_zone(&self, is_leaf: bool, frame_size: u32) -> bool {
if !self.has_red_zone {
return false;
}
if !is_leaf {
return false;
}
frame_size <= RED_ZONE_SIZE
}
pub fn can_eliminate_frame_pointer(
&self,
has_var_sized_objects: bool,
has_calls: bool,
frame_size: u32,
) -> bool {
if has_var_sized_objects {
return false; }
if has_calls && frame_size > 0 {
return !self.uses_seh;
}
true
}
pub fn needs_stack_probe(&self, frame_size: u32) -> StackProbeKind {
if frame_size > self.stack_probe_threshold {
if self.target_variant.is_64bit_mode() && self.uses_seh {
StackProbeKind::ChkStk
} else {
StackProbeKind::Inline
}
} else {
StackProbeKind::None
}
}
pub fn emit_stack_probe(&mut self, frame_size: u32) -> Vec<X86MachineInstr> {
let kind = self.needs_stack_probe(frame_size);
match kind {
StackProbeKind::ChkStk => {
vec![X86MachineInstr {
opcode: MOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
defs: vec![],
uses: vec![X86MachineOperand::imm(frame_size as i64)],
flags: X86MIFlags {
is_call: true,
has_side_effects: true,
..Default::default()
},
}]
}
StackProbeKind::Inline => {
vec![X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![
X86MachineOperand::vreg(VReg(self.sp_reg)),
X86MachineOperand::imm(0),
],
flags: X86MIFlags {
may_store: true,
..Default::default()
},
}]
}
StackProbeKind::None => Vec::new(),
_ => Vec::new(),
}
}
pub fn emit_realignment_prologue(&mut self, alignment: u32) -> Vec<X86MachineInstr> {
let mut instrs = Vec::new();
instrs.push(X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(self.fp_reg))],
flags: X86MIFlags {
may_store: true,
..Default::default()
},
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BITCAST,
defs: vec![VReg(self.fp_reg)],
uses: vec![X86MachineOperand::vreg(VReg(self.sp_reg))],
flags: X86MIFlags::default(),
});
let mask = -(alignment as i64);
instrs.push(X86MachineInstr {
opcode: MOpcode::G_AND,
defs: vec![VReg(self.sp_reg)],
uses: vec![
X86MachineOperand::vreg(VReg(self.sp_reg)),
X86MachineOperand::imm(mask),
],
flags: X86MIFlags::default(),
});
instrs
}
pub fn emit_prologue_32(&mut self, frame_size: u32) -> Vec<X86MachineInstr> {
let mut instrs = Vec::new();
if self.use_frame_pointer {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(5))], flags: X86MIFlags {
may_store: true,
..Default::default()
},
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BITCAST,
defs: vec![VReg(5)], uses: vec![X86MachineOperand::vreg(VReg(4))], flags: X86MIFlags::default(),
});
if frame_size > 0 {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_SUB,
defs: vec![VReg(4)], uses: vec![
X86MachineOperand::vreg(VReg(4)),
X86MachineOperand::imm(frame_size as i64),
],
flags: X86MIFlags::default(),
});
}
}
instrs
}
pub fn emit_epilogue_32(&mut self, _frame_size: u32) -> Vec<X86MachineInstr> {
let mut instrs = Vec::new();
if self.use_frame_pointer {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BITCAST,
defs: vec![VReg(4)], uses: vec![X86MachineOperand::vreg(VReg(5))], flags: X86MIFlags::default(),
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_LOAD,
defs: vec![VReg(5)], uses: vec![X86MachineOperand::vreg(VReg(4))], flags: X86MIFlags {
may_load: true,
..Default::default()
},
});
}
instrs.push(X86MachineInstr {
opcode: MOpcode::G_RETURN,
defs: vec![],
uses: vec![],
flags: X86MIFlags {
is_terminator: true,
is_return: true,
is_barrier: true,
..Default::default()
},
});
instrs
}
pub fn should_eliminate_frame_pointer(
&self,
opt_level: X86OptLevel,
has_var_sized: bool,
has_calls: bool,
frame_size: u32,
) -> FramePointerElimInfo {
if opt_level == X86OptLevel::O0 {
return FramePointerElimInfo {
eliminated: false,
reason: Some(FpElimReason::DebugInfo),
sp_offset: 0,
};
}
if has_var_sized {
return FramePointerElimInfo {
eliminated: false,
reason: Some(FpElimReason::VarSizedObjects),
sp_offset: 0,
};
}
if has_calls && frame_size > 0 && self.uses_seh {
return FramePointerElimInfo {
eliminated: false,
reason: Some(FpElimReason::ExplicitRequest),
sp_offset: 0,
};
}
FramePointerElimInfo {
eliminated: true,
reason: Some(FpElimReason::ExplicitRequest),
sp_offset: frame_size as i64,
}
}
}
impl Default for X86FrameLoweringEngine {
fn default() -> Self {
Self::new(
X86TargetVariant::default(),
X86CallingConvention::X86_64_SysV,
)
}
}
#[derive(Debug, Clone, Default)]
pub struct X86CalleeSavedInfo {
pub spill_slots: Vec<SpillSlot>,
pub total_size: u32,
pub regs_to_save: Vec<u32>,
pub shrink_wrap_profitable: bool,
pub shrink_wrap_save_block: Option<u32>,
pub shrink_wrap_restore_block: Option<u32>,
pub xmm_regs_to_save: Vec<u32>,
pub xmm_save_size: u32,
}
pub struct X86CalleeSaved {
pub target_variant: X86TargetVariant,
pub callee_saved_regs: Vec<u32>,
pub shrink_wrappable_regs: Vec<u32>,
}
impl X86CalleeSaved {
pub fn new(target_variant: X86TargetVariant) -> Self {
let callee_saved = if target_variant.is_64bit_mode() {
SYSV64_CALLEE_SAVED_GPRS.to_vec()
} else {
X86_32_CALLEE_SAVED_GPRS.to_vec()
};
X86CalleeSaved {
target_variant,
callee_saved_regs: callee_saved,
shrink_wrappable_regs: vec![3, 12, 13, 14, 15], }
}
pub fn compute_spills(&self, frame: &X86FrameLayout) -> X86CalleeSavedInfo {
let mut info = X86CalleeSavedInfo::default();
for ® in &self.callee_saved_regs {
if reg == 3 || reg == 5 {
info.regs_to_save.push(reg);
let slot_size = (self.target_variant.register_width() / 8) as i64;
let slot = SpillSlot {
vreg: reg,
offset: (frame.callee_saved_start + info.total_size) as i64,
size: slot_size,
alignment: slot_size,
reg_class: SpillSlotClass::Gpr64,
fixed: false,
};
info.total_size += slot_size as u32;
info.spill_slots.push(slot);
}
}
info.total_size = align_to(info.total_size, self.target_variant.register_width() / 8);
info
}
pub fn assign_spill_slots(&mut self, regs: &[u32], start_offset: u32) -> Vec<SpillSlot> {
let mut slots = Vec::new();
let mut offset = start_offset;
for ® in regs {
let size = if self.target_variant.is_64bit_mode() {
8
} else {
4
};
let slot = SpillSlot {
vreg: reg,
offset: offset as i64,
size: size as i64,
alignment: size as i64,
reg_class: if self.target_variant.is_64bit_mode() {
SpillSlotClass::Gpr64
} else {
SpillSlotClass::Gpr32
},
fixed: false,
};
slots.push(slot);
offset += size as u32;
}
slots
}
pub fn is_shrink_wrap_eligible(&self, has_calls: bool, saved_regs: &[u32]) -> ShrinkWrapInfo {
if !has_calls {
return ShrinkWrapInfo {
profitable: false,
save_block: None,
restore_block: None,
shrinkable_regs: Vec::new(),
};
}
let shrinkable: Vec<u32> = saved_regs
.iter()
.filter(|r| self.shrink_wrappable_regs.contains(r))
.cloned()
.collect();
if shrinkable.is_empty() {
return ShrinkWrapInfo {
profitable: false,
save_block: None,
restore_block: None,
shrinkable_regs: Vec::new(),
};
}
ShrinkWrapInfo {
profitable: true,
save_block: Some(0), restore_block: Some(1), shrinkable_regs: shrinkable.into_iter().map(|r| r as u16).collect(),
}
}
pub fn get_callee_saved_gprs(&self) -> Vec<u32> {
self.callee_saved_regs.clone()
}
pub fn get_callee_saved_xmms_sysv64(&self) -> Vec<u32> {
vec![]
}
pub fn get_callee_saved_xmms_win64(&self) -> Vec<u32> {
(38..=47).collect() }
pub fn with_calling_convention(mut self, cc: X86CallingConvention) -> Self {
self.callee_saved_regs = match cc {
X86CallingConvention::Win64 => WIN64_CALLEE_SAVED_GPRS.to_vec(),
X86CallingConvention::X86_64_SysV => SYSV64_CALLEE_SAVED_GPRS.to_vec(),
_ => {
if self.target_variant.is_64bit_mode() {
SYSV64_CALLEE_SAVED_GPRS.to_vec()
} else {
X86_32_CALLEE_SAVED_GPRS.to_vec()
}
}
};
self
}
}
impl Default for X86CalleeSaved {
fn default() -> Self {
Self::new(X86TargetVariant::default())
}
}
pub struct X86ShadowStack {
pub target_variant: X86TargetVariant,
pub enabled: bool,
pub insert_endbr: bool,
pub prologue: Vec<X86MachineInstr>,
pub epilogue: Vec<X86MachineInstr>,
}
impl X86ShadowStack {
pub fn new(target_variant: X86TargetVariant) -> Self {
X86ShadowStack {
target_variant,
enabled: false,
insert_endbr: true,
prologue: Vec::new(),
epilogue: Vec::new(),
}
}
pub fn lower(&mut self) {
if !self.enabled {
return;
}
if self.insert_endbr {
self.prologue.push(X86MachineInstr {
opcode: MOpcode::G_INTRINSIC, defs: vec![],
uses: vec![],
flags: X86MIFlags {
has_side_effects: false,
is_barrier: true,
..Default::default()
},
});
}
self.prologue.push(X86MachineInstr {
opcode: MOpcode::G_INTRINSIC, defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(4))], flags: X86MIFlags {
has_side_effects: true,
..Default::default()
},
});
self.prologue.push(X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(4))], flags: X86MIFlags {
may_store: true,
has_side_effects: true,
..Default::default()
},
});
self.epilogue.push(X86MachineInstr {
opcode: MOpcode::G_INTRINSIC, defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(4))],
flags: X86MIFlags {
has_side_effects: true,
..Default::default()
},
});
}
pub fn with_shadow_stack(mut self, enable: bool) -> Self {
self.enabled = enable;
self
}
pub fn with_endbr(mut self, enable: bool) -> Self {
self.insert_endbr = enable;
self
}
}
impl Default for X86ShadowStack {
fn default() -> Self {
Self::new(X86TargetVariant::default())
}
}
pub struct X86EHPadLowering {
pub target_variant: X86TargetVariant,
pub personality_fn: Option<String>,
pub uses_eh: bool,
pub landing_pads: Vec<LoweredLandingPad>,
pub eh_table_builder: Option<ExceptionTableBuilder>,
}
#[derive(Debug, Clone)]
pub struct LoweredLandingPad {
pub block_id: u32,
pub catches: Vec<LoweredCatchClause>,
pub filters: Vec<LoweredFilterClause>,
pub has_cleanup: bool,
pub cleanup_block: Option<u32>,
pub resume_block: Option<u32>,
}
#[derive(Debug, Clone)]
pub struct LoweredCatchClause {
pub exception_type: String,
pub handler_block: u32,
pub is_catch_all: bool,
}
#[derive(Debug, Clone)]
pub struct LoweredFilterClause {
pub filter_value: i32,
pub handler_block: u32,
}
impl X86EHPadLowering {
pub fn new(target_variant: X86TargetVariant) -> Self {
X86EHPadLowering {
target_variant,
personality_fn: None,
uses_eh: false,
landing_pads: Vec::new(),
eh_table_builder: None,
}
}
pub fn lower_landing_pads(&mut self, pads: &[LandingPadDescriptor]) {
self.uses_eh = !pads.is_empty();
for pad in pads {
let mut lowered = LoweredLandingPad {
block_id: pad.block_id,
catches: Vec::new(),
filters: Vec::new(),
has_cleanup: pad.cleanup,
cleanup_block: None,
resume_block: None,
};
for &catch_type in &pad.catch_types {
lowered.catches.push(LoweredCatchClause {
exception_type: format!("type_{}", catch_type),
handler_block: pad.block_id + 1,
is_catch_all: catch_type == 0,
});
}
for &filter_type in &pad.filter_types {
lowered.filters.push(LoweredFilterClause {
filter_value: filter_type as i32,
handler_block: pad.block_id + 1,
});
}
if pad.cleanup {
lowered.cleanup_block = Some(pad.block_id + 2);
}
lowered.resume_block = Some(pad.block_id + 3);
self.landing_pads.push(lowered);
}
}
pub fn generate_dispatch(&self, pad_index: usize) -> Vec<X86MachineInstr> {
let mut instrs = Vec::new();
if let Some(pad) = self.landing_pads.get(pad_index) {
for (i, catch) in pad.catches.iter().enumerate() {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_ICMP,
defs: vec![VReg(100 + i as u32)],
uses: vec![
X86MachineOperand::vreg(VReg(99)), X86MachineOperand::imm(i as i64 + 1),
],
flags: X86MIFlags {
is_compare: true,
..Default::default()
},
});
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BRCOND,
defs: vec![],
uses: vec![
X86MachineOperand::vreg(VReg(100 + i as u32)),
X86MachineOperand::block(catch.handler_block),
],
flags: X86MIFlags {
is_terminator: false,
is_branch: true,
..Default::default()
},
});
}
if pad.has_cleanup {
if let Some(cleanup_block) = pad.cleanup_block {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_BR,
defs: vec![],
uses: vec![X86MachineOperand::block(cleanup_block)],
flags: X86MIFlags {
is_terminator: true,
is_branch: true,
..Default::default()
},
});
}
}
if let Some(resume_block) = pad.resume_block {
instrs.push(X86MachineInstr {
opcode: MOpcode::G_RESUME,
defs: vec![],
uses: vec![X86MachineOperand::block(resume_block)],
flags: X86MIFlags {
is_terminator: true,
..Default::default()
},
});
}
}
instrs
}
pub fn generate_resume(&self) -> Vec<X86MachineInstr> {
vec![X86MachineInstr {
opcode: MOpcode::G_RESUME,
defs: vec![],
uses: vec![],
flags: X86MIFlags {
is_terminator: true,
is_barrier: true,
..Default::default()
},
}]
}
pub fn generate_cleanup(&self) -> Vec<X86MachineInstr> {
vec![X86MachineInstr {
opcode: MOpcode::G_CLEANUPRET,
defs: vec![],
uses: vec![],
flags: X86MIFlags {
is_terminator: true,
..Default::default()
},
}]
}
pub fn with_personality(mut self, personality: &str) -> Self {
self.personality_fn = Some(personality.to_string());
self
}
}
impl Default for X86EHPadLowering {
fn default() -> Self {
Self::new(X86TargetVariant::default())
}
}
#[derive(Debug, Clone)]
pub struct X86CFIEntry {
pub is_cie: bool,
pub length: u32,
pub cie_id: u32,
pub version: u8,
pub augmentation: String,
pub code_alignment_factor: u32,
pub data_alignment_factor: i32,
pub return_address_register: u32,
pub initial_cfa_offset: u32,
pub initial_cfa_register: u32,
pub initial_instructions: Vec<u8>,
pub pc_begin: Option<u64>,
pub pc_range: Option<u32>,
pub instructions: Vec<u8>,
}
impl Default for X86CFIEntry {
fn default() -> Self {
X86CFIEntry {
is_cie: false,
length: 0,
cie_id: 0,
version: 1,
augmentation: String::new(),
code_alignment_factor: 1,
data_alignment_factor: -8,
return_address_register: DWARF_RETURN_ADDR,
initial_cfa_offset: 0,
initial_cfa_register: DWARF_RSP,
initial_instructions: Vec::new(),
pc_begin: None,
pc_range: None,
instructions: Vec::new(),
}
}
}
pub struct X86DwarfCFI {
pub target_variant: X86TargetVariant,
pub enabled: bool,
pub cie: X86CFIEntry,
pub fdes: Vec<X86CFIEntry>,
pub current_cfa_offset: u32,
pub current_cfa_register: u32,
pub register_save_locations: Vec<(u32, i32, u32)>,
pub directive_count: u32,
}
impl X86DwarfCFI {
pub fn new(target_variant: X86TargetVariant) -> Self {
let ptr_size = target_variant.pointer_width() / 8;
let data_align = if target_variant.is_64bit_mode() {
-8i32
} else {
-4i32
};
let ra_reg = if target_variant.is_64bit_mode() {
DWARF_RETURN_ADDR
} else {
8u32
};
X86DwarfCFI {
target_variant,
enabled: true,
cie: X86CFIEntry {
is_cie: true,
version: 1,
augmentation: "zR".to_string(),
code_alignment_factor: 1,
data_alignment_factor: data_align,
return_address_register: ra_reg,
initial_cfa_register: DWARF_RSP,
initial_cfa_offset: ptr_size, ..Default::default()
},
fdes: Vec::new(),
current_cfa_offset: 0,
current_cfa_register: DWARF_RSP,
register_save_locations: Vec::new(),
directive_count: 0,
}
}
pub fn emit_cfi(&mut self) {
if !self.enabled {
return;
}
self.emit_def_cfa(DWARF_RSP, self.cie.initial_cfa_offset);
self.directive_count += 1;
self.current_cfa_offset = self.cie.initial_cfa_offset;
self.current_cfa_register = DWARF_RSP;
}
pub fn emit_def_cfa(&mut self, register: u32, offset: u32) {
self.current_cfa_register = register;
self.current_cfa_offset = offset;
self.directive_count += 1;
}
pub fn emit_def_cfa_offset(&mut self, offset: u32) {
self.current_cfa_offset = offset;
self.directive_count += 1;
}
pub fn emit_def_cfa_register(&mut self, register: u32) {
self.current_cfa_register = register;
self.directive_count += 1;
}
pub fn emit_offset(&mut self, register: u32, offset: i32) {
self.register_save_locations
.push((register, offset, register));
self.directive_count += 1;
}
pub fn emit_same_value(&mut self, register: u32) {
self.register_save_locations.push((register, 0, register));
self.directive_count += 1;
}
pub fn emit_restore(&mut self, register: u32) {
self.register_save_locations
.retain(|(r, _, _)| *r != register);
self.directive_count += 1;
}
pub fn begin_fde(&mut self, pc_begin: u64, pc_range: u32) {
let mut fde = X86CFIEntry {
is_cie: false,
version: self.cie.version,
augmentation: self.cie.augmentation.clone(),
code_alignment_factor: self.cie.code_alignment_factor,
data_alignment_factor: self.cie.data_alignment_factor,
return_address_register: self.cie.return_address_register,
pc_begin: Some(pc_begin),
pc_range: Some(pc_range),
..Default::default()
};
fde.initial_cfa_register = self.cie.initial_cfa_register;
fde.initial_cfa_offset = self.cie.initial_cfa_offset;
self.fdes.push(fde);
self.directive_count = 0;
}
pub fn end_fde(&mut self) {
if let Some(fde) = self.fdes.last_mut() {
fde.instructions = vec![0; 0]; }
}
pub fn emit_cie_bytes(&self) -> Vec<u8> {
let mut bytes = Vec::new();
let length: u32 = 0; bytes.extend_from_slice(&length.to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
bytes.push(self.cie.version);
bytes.extend_from_slice(self.cie.augmentation.as_bytes());
bytes.push(0);
bytes.extend(encode_uleb128(self.cie.code_alignment_factor as u64));
bytes.extend(encode_sleb128(self.cie.data_alignment_factor as i64));
bytes.extend(encode_uleb128(self.cie.return_address_register as u64));
bytes.extend_from_slice(&self.cie.initial_instructions);
bytes
}
pub fn emit_fde_bytes(&self, fde_index: usize) -> Vec<u8> {
if let Some(fde) = self.fdes.get(fde_index) {
let mut bytes = Vec::new();
let length: u32 = 0;
bytes.extend_from_slice(&length.to_le_bytes());
bytes.extend_from_slice(&0u32.to_le_bytes());
if let Some(pc_begin) = fde.pc_begin {
bytes.extend_from_slice(&pc_begin.to_le_bytes());
}
if let Some(pc_range) = fde.pc_range {
bytes.extend_from_slice(&pc_range.to_le_bytes());
}
bytes.extend_from_slice(&fde.instructions);
bytes
} else {
Vec::new()
}
}
pub fn directive_count(&self) -> u32 {
self.directive_count
}
pub fn is_enabled(&self) -> bool {
self.enabled
}
pub fn with_cfi(mut self, enable: bool) -> Self {
self.enabled = enable;
self
}
}
impl Default for X86DwarfCFI {
fn default() -> Self {
Self::new(X86TargetVariant::default())
}
}
fn align_to(value: u32, alignment: u32) -> u32 {
if alignment == 0 {
return value;
}
(value + alignment - 1) & !(alignment - 1)
}
fn encode_uleb128(mut value: u64) -> Vec<u8> {
let mut result = Vec::new();
loop {
let mut byte = (value & 0x7f) as u8;
value >>= 7;
if value != 0 {
byte |= 0x80;
}
result.push(byte);
if value == 0 {
break;
}
}
result
}
fn encode_sleb128(mut value: i64) -> Vec<u8> {
let mut result = Vec::new();
loop {
let mut byte = (value & 0x7f) as u8;
value >>= 7;
if (value == 0 && (byte & 0x40) == 0) || (value == -1 && (byte & 0x40) != 0) {
result.push(byte);
break;
}
byte |= 0x80;
result.push(byte);
}
result
}
pub fn fits_i8(value: i64) -> bool {
value >= -128 && value <= 127
}
pub fn fits_i32(value: i64) -> bool {
value >= -2147483648 && value <= 2147483647
}
pub fn is_imm8(value: i64) -> bool {
fits_i8(value)
}
pub fn is_imm32(value: i64) -> bool {
fits_i32(value)
}
pub fn type_size_bytes(type_id: u32) -> u32 {
match type_id {
0 => 0, 1 => 1, 8 => 1, 16 => 2, 32 => 4, 64 => 8, 33 => 4, 65 => 8, 80 => 10, 128 => 16, _ => 8, }
}
pub fn create_x86_64_sysv_lowering() -> X86Lowering {
X86Lowering::new_x86_64_sysv()
}
pub fn create_x86_64_win64_lowering() -> X86Lowering {
X86Lowering::new_x86_64_win64()
}
pub fn create_x86_32_cdecl_lowering() -> X86Lowering {
X86Lowering::new_x86_32_cdecl()
}
pub fn create_x86_ir_transformer(target_variant: X86TargetVariant) -> X86IRTransformer {
X86IRTransformer::new(target_variant)
}
pub fn create_x86_call_lowering(
target_variant: X86TargetVariant,
cc: X86CallingConvention,
) -> X86CallLowering {
X86CallLowering::new(target_variant, cc)
}
pub fn create_x86_arg_lowering(
target_variant: X86TargetVariant,
cc: X86CallingConvention,
) -> X86ArgLowering {
X86ArgLowering::new(target_variant, cc)
}
pub fn create_x86_ret_lowering(
target_variant: X86TargetVariant,
cc: X86CallingConvention,
) -> X86RetLowering {
X86RetLowering::new(target_variant, cc)
}
pub fn create_x86_frame_lowering(
target_variant: X86TargetVariant,
cc: X86CallingConvention,
) -> X86FrameLoweringEngine {
X86FrameLoweringEngine::new(target_variant, cc)
}
pub fn create_x86_callee_saved(target_variant: X86TargetVariant) -> X86CalleeSaved {
X86CalleeSaved::new(target_variant)
}
pub fn create_x86_shadow_stack(target_variant: X86TargetVariant) -> X86ShadowStack {
X86ShadowStack::new(target_variant)
}
pub fn create_x86_eh_pad_lowering(target_variant: X86TargetVariant) -> X86EHPadLowering {
X86EHPadLowering::new(target_variant)
}
pub fn create_x86_dwarf_cfi(target_variant: X86TargetVariant) -> X86DwarfCFI {
X86DwarfCFI::new(target_variant)
}
#[cfg(test)]
mod tests {
use super::*;
fn make_test_arg(size: u32) -> X86LoweringArg {
X86LoweringArg {
size,
alignment: if size <= 8 { size } else { 8 },
..Default::default()
}
}
fn make_test_ret(size: u32, is_fp: bool) -> X86LoweringRet {
if is_fp {
X86LoweringRet {
regs: vec![X86LoweringReg::xmm(0)],
size,
..Default::default()
}
} else {
X86LoweringRet {
regs: vec![X86LoweringReg::gpr64(0)],
size,
..Default::default()
}
}
}
#[test]
fn test_target_variant_is_64bit() {
assert!(X86TargetVariant::X86_64.is_64bit_mode());
assert!(!X86TargetVariant::X86_32.is_64bit_mode());
assert!(X86TargetVariant::X32.is_64bit_mode());
assert!(!X86TargetVariant::X86_16.is_64bit_mode());
}
#[test]
fn test_target_variant_pointer_width() {
assert_eq!(X86TargetVariant::X86_64.pointer_width(), 64);
assert_eq!(X86TargetVariant::X86_32.pointer_width(), 32);
assert_eq!(X86TargetVariant::X32.pointer_width(), 32);
assert_eq!(X86TargetVariant::X86_16.pointer_width(), 16);
}
#[test]
fn test_target_variant_register_width() {
assert_eq!(X86TargetVariant::X86_64.register_width(), 64);
assert_eq!(X86TargetVariant::X86_32.register_width(), 32);
assert_eq!(X86TargetVariant::X32.register_width(), 64);
assert_eq!(X86TargetVariant::X86_16.register_width(), 16);
}
#[test]
fn test_target_variant_default() {
assert_eq!(X86TargetVariant::default(), X86TargetVariant::X86_64);
}
#[test]
fn test_lowering_reg_gpr64() {
let reg = X86LoweringReg::gpr64(0); assert_eq!(reg.reg_num, 0);
assert_eq!(reg.reg_class, X86LoweringRegClass::GPR64);
assert!(reg.is_call_clobbered_sysv64());
assert!(reg.is_call_clobbered_win64());
}
#[test]
fn test_lowering_reg_xmm() {
let reg = X86LoweringReg::xmm(0); assert_eq!(reg.reg_num, 0);
assert_eq!(reg.reg_class, X86LoweringRegClass::XMM);
assert!(reg.is_call_clobbered_sysv64());
}
#[test]
fn test_lowering_reg_not_call_clobbered() {
let reg = X86LoweringReg::gpr64(3); assert!(!reg.is_call_clobbered_sysv64());
assert!(!reg.is_call_clobbered_win64());
}
#[test]
fn test_lowering_reg_r12_is_callee_saved() {
let reg = X86LoweringReg::gpr64(12); assert!(!reg.is_call_clobbered_sysv64());
}
#[test]
fn test_reg_class_size_bytes() {
assert_eq!(X86LoweringRegClass::GPR8.size_bytes(), 1);
assert_eq!(X86LoweringRegClass::GPR16.size_bytes(), 2);
assert_eq!(X86LoweringRegClass::GPR32.size_bytes(), 4);
assert_eq!(X86LoweringRegClass::GPR64.size_bytes(), 8);
assert_eq!(X86LoweringRegClass::XMM.size_bytes(), 16);
assert_eq!(X86LoweringRegClass::YMM.size_bytes(), 32);
assert_eq!(X86LoweringRegClass::ZMM.size_bytes(), 64);
assert_eq!(X86LoweringRegClass::X87.size_bytes(), 10);
}
#[test]
fn test_reg_class_is_gpr() {
assert!(X86LoweringRegClass::GPR8.is_gpr());
assert!(X86LoweringRegClass::GPR16.is_gpr());
assert!(X86LoweringRegClass::GPR32.is_gpr());
assert!(X86LoweringRegClass::GPR64.is_gpr());
assert!(!X86LoweringRegClass::XMM.is_gpr());
}
#[test]
fn test_reg_class_is_vector() {
assert!(X86LoweringRegClass::XMM.is_vector());
assert!(X86LoweringRegClass::YMM.is_vector());
assert!(X86LoweringRegClass::ZMM.is_vector());
assert!(!X86LoweringRegClass::GPR64.is_vector());
}
#[test]
fn test_vreg_creation() {
let v = VReg::new(42);
assert_eq!(v.0, 42);
}
#[test]
fn test_vreg_equality() {
assert_eq!(VReg(1), VReg(1));
assert_ne!(VReg(1), VReg(2));
}
#[test]
fn test_frame_index_creation() {
let fi = FrameIndex::new(10);
assert_eq!(fi.0, 10);
}
#[test]
fn test_machine_operand_vreg() {
let op = X86MachineOperand::vreg(VReg(5));
match op {
X86MachineOperand::VReg(v) => assert_eq!(v.0, 5),
_ => panic!("Expected VReg"),
}
}
#[test]
fn test_machine_operand_imm() {
let op = X86MachineOperand::imm(-42);
match op {
X86MachineOperand::Imm(v) => assert_eq!(v, -42),
_ => panic!("Expected Imm"),
}
}
#[test]
fn test_machine_operand_block() {
let op = X86MachineOperand::block(7);
match op {
X86MachineOperand::Block(b) => assert_eq!(b.0, 7),
_ => panic!("Expected Block"),
}
}
#[test]
fn test_machine_operand_frame_index() {
let op = X86MachineOperand::frame_index(3);
match op {
X86MachineOperand::FrameIndex(fi) => assert_eq!(fi.0, 3),
_ => panic!("Expected FrameIndex"),
}
}
#[test]
fn test_mi_flags_default() {
let flags = X86MIFlags::default();
assert!(!flags.is_terminator);
assert!(!flags.is_branch);
assert!(!flags.is_call);
assert!(!flags.is_return);
assert!(!flags.may_load);
assert!(!flags.may_store);
assert!(!flags.has_side_effects);
assert!(!flags.is_barrier);
}
#[test]
fn test_mi_flags_return() {
let flags = X86MIFlags {
is_terminator: true,
is_return: true,
is_barrier: true,
..Default::default()
};
assert!(flags.is_terminator);
assert!(flags.is_return);
assert!(flags.is_barrier);
assert!(!flags.is_call);
}
#[test]
fn test_lowering_new_default() {
let lowering = X86Lowering::default();
assert_eq!(lowering.target_variant, X86TargetVariant::X86_64);
assert_eq!(
lowering.calling_convention,
X86CallingConvention::X86_64_SysV
);
assert_eq!(lowering.mode, X86LoweringMode::SelectionDAG);
assert_eq!(lowering.opt_level, X86OptLevel::O0);
assert!(lowering.has_sse2);
assert!(!lowering.has_avx);
assert!(!lowering.has_avx512);
assert!(lowering.has_cmov);
assert!(!lowering.has_cet);
assert!(lowering.eliminate_frame_pointer);
assert!(lowering.use_red_zone);
}
#[test]
fn test_lowering_new_x86_64_sysv() {
let lowering = X86Lowering::new_x86_64_sysv();
assert_eq!(lowering.target_variant, X86TargetVariant::X86_64);
assert_eq!(
lowering.calling_convention,
X86CallingConvention::X86_64_SysV
);
assert!(lowering.use_red_zone);
}
#[test]
fn test_lowering_new_win64() {
let lowering = X86Lowering::new_x86_64_win64();
assert_eq!(lowering.target_variant, X86TargetVariant::X86_64);
assert_eq!(lowering.calling_convention, X86CallingConvention::Win64);
assert!(!lowering.use_red_zone);
}
#[test]
fn test_lowering_new_x86_32_cdecl() {
let lowering = X86Lowering::new_x86_32_cdecl();
assert_eq!(lowering.target_variant, X86TargetVariant::X86_32);
assert_eq!(lowering.calling_convention, X86CallingConvention::C);
assert!(!lowering.use_red_zone);
}
#[test]
fn test_lowering_with_calling_convention() {
let lowering = X86Lowering::default().with_calling_convention(X86CallingConvention::Win64);
assert_eq!(lowering.calling_convention, X86CallingConvention::Win64);
assert_eq!(
lowering.call_lowering.calling_convention,
X86CallingConvention::Win64
);
}
#[test]
fn test_lowering_with_opt_level() {
let lowering = X86Lowering::default().with_opt_level(X86OptLevel::O3);
assert_eq!(lowering.opt_level, X86OptLevel::O3);
}
#[test]
fn test_lowering_with_mode() {
let lowering = X86Lowering::default().with_mode(X86LoweringMode::GlobalISel);
assert_eq!(lowering.mode, X86LoweringMode::GlobalISel);
}
#[test]
fn test_lowering_with_avx() {
let lowering = X86Lowering::default().with_avx(true);
assert!(lowering.has_avx);
}
#[test]
fn test_lowering_with_avx512() {
let lowering = X86Lowering::default().with_avx512(true);
assert!(lowering.has_avx512);
assert!(lowering.has_avx); }
#[test]
fn test_lowering_with_cet() {
let lowering = X86Lowering::default().with_cet(true);
assert!(lowering.has_cet);
}
#[test]
fn test_lowering_with_frame_pointer_elim() {
let lowering = X86Lowering::default().with_frame_pointer_elim(false);
assert!(!lowering.eliminate_frame_pointer);
}
#[test]
fn test_lowering_with_red_zone() {
let lowering = X86Lowering::default().with_red_zone(false);
assert!(!lowering.use_red_zone);
}
#[test]
fn test_lowering_begin_finish_function() {
let mut lowering = X86Lowering::default();
lowering.begin_function("test_func");
assert!(lowering.function_state.is_some());
assert_eq!(lowering.function_state.as_ref().unwrap().name, "test_func");
let state = lowering.finish_function();
assert!(state.is_some());
assert_eq!(lowering.stats.functions_lowered, 1);
assert!(lowering.function_state.is_none());
}
#[test]
fn test_lowering_lower_function() {
let mut lowering = X86Lowering::default();
let args = vec![make_test_arg(8), make_test_arg(4)];
let ret = Some(make_test_ret(8, false));
let result = lowering.lower_function("test", &args, ret.as_ref());
assert!(result.function_state.is_some());
assert_eq!(result.args_lowered.len(), 2);
assert!(result.ret_lowered.is_some());
assert_eq!(result.stats.functions_lowered, 1);
assert_eq!(result.stats.args_lowered, 2);
assert_eq!(result.stats.returns_lowered, 1);
}
#[test]
fn test_lowering_get_callee_saved_regs() {
let lowering = X86Lowering::default();
let regs = lowering.get_callee_saved_regs();
assert!(regs.is_empty()); }
#[test]
fn test_lowering_is_tail_call_eligible() {
let lowering = X86Lowering::default();
let eligible = lowering.is_tail_call_eligible(
X86CallingConvention::X86_64_SysV,
X86CallingConvention::X86_64_SysV,
true,
);
assert!(eligible);
}
#[test]
fn test_lowering_is_tail_call_eligible_cross_abi() {
let lowering = X86Lowering::default();
let eligible = lowering.is_tail_call_eligible(
X86CallingConvention::X86_64_SysV,
X86CallingConvention::Win64,
false,
);
assert!(!eligible);
}
#[test]
fn test_ir_transformer_new() {
let transformer = X86IRTransformer::new(X86TargetVariant::X86_64);
assert!(transformer.supports_expansion("llvm.memcpy"));
assert!(transformer.supports_expansion("llvm.sqrt.f64"));
assert!(transformer.supports_expansion("llvm.x86.sse2.add.sd"));
assert!(!transformer.supports_expansion("nonexistent.intrinsic"));
}
#[test]
fn test_ir_transformer_transform() {
let mut transformer = X86IRTransformer::new(X86TargetVariant::X86_64);
let result = transformer.transform();
assert!(result.instructions_transformed > 0);
assert!(result.intrinsics_expanded > 0);
assert!(result.target_intrinsics_expanded > 0);
}
#[test]
fn test_ir_transformer_default() {
let transformer = X86IRTransformer::default();
assert_eq!(transformer.target_variant, X86TargetVariant::X86_64);
assert!(transformer.expand_mem_intrinsics);
assert!(transformer.expand_math_intrinsics);
assert!(transformer.lower_varargs);
}
#[test]
fn test_ir_transformer_with_expand_mem() {
let transformer = X86IRTransformer::default().with_expand_mem(false);
assert!(!transformer.expand_mem_intrinsics);
}
#[test]
fn test_ir_transformer_with_expand_math() {
let transformer = X86IRTransformer::default().with_expand_math(false);
assert!(!transformer.expand_math_intrinsics);
}
#[test]
fn test_ir_transformer_with_expand_atomic() {
let transformer = X86IRTransformer::default().with_expand_atomic(true);
assert!(transformer.expand_atomic_intrinsics);
}
#[test]
fn test_ir_transformer_with_lower_varargs() {
let transformer = X86IRTransformer::default().with_lower_varargs(false);
assert!(!transformer.lower_varargs);
}
#[test]
fn test_ir_transformer_supported_intrinsics_count() {
let transformer = X86IRTransformer::default();
assert!(transformer.supported_expansions.len() > 50);
}
#[test]
fn test_call_lowering_new_sysv64() {
let cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(cl.calling_convention, X86CallingConvention::X86_64_SysV);
assert_eq!(cl.shadow_store_size, 0);
assert_eq!(cl.stack_alignment, 16);
assert!(cl.enable_tail_calls);
}
#[test]
fn test_call_lowering_new_win64() {
let cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert_eq!(cl.shadow_store_size, 32);
assert_eq!(cl.stack_alignment, 16);
}
#[test]
fn test_call_lowering_new_32bit() {
let cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
assert_eq!(cl.shadow_store_size, 0);
}
#[test]
fn test_call_lowering_default() {
let cl = X86CallLowering::default();
assert_eq!(cl.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_call_lowering_lower_sysv64_call() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let args = vec![make_test_arg(8), make_test_arg(4)];
let ret = Some(make_test_ret(8, false));
let site = cl.lower_sysv64_call("target_func", &args, ret.as_ref());
assert_eq!(site.callee, "target_func");
assert!(!site.arg_assignments.is_empty());
}
#[test]
fn test_call_lowering_lower_win64_call() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let args = vec![make_test_arg(8)];
let ret = Some(make_test_ret(8, false));
let site = cl.lower_win64_call("win_target", &args, ret.as_ref());
assert_eq!(site.callee, "win_target");
assert!(site.arg_stack_size >= WIN64_SHADOW_SPACE);
}
#[test]
fn test_call_lowering_lower_cdecl_call() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let args = vec![make_test_arg(4), make_test_arg(4)];
let ret = Some(make_test_ret(4, false));
let site = cl.lower_cdecl_call("cdecl_func", &args, ret.as_ref());
assert_eq!(site.callee, "cdecl_func");
assert!(site.arg_stack_size > 0);
}
#[test]
fn test_call_lowering_lower_fastcall_call() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::Fast);
let args = vec![make_test_arg(4), make_test_arg(4), make_test_arg(4)];
let ret = Some(make_test_ret(4, false));
let site = cl.lower_fastcall_call("fastcall_func", &args, ret.as_ref());
assert_eq!(site.callee, "fastcall_func");
assert_eq!(site.call_conv_override, Some(X86CallingConvention::Fast));
}
#[test]
fn test_call_lowering_lower_thiscall_call() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::ThisCall);
let args = vec![make_test_arg(4), make_test_arg(4)];
let ret = Some(make_test_ret(4, false));
let site = cl.lower_thiscall_call("method", &args, ret.as_ref());
assert_eq!(site.callee, "method");
assert_eq!(
site.call_conv_override,
Some(X86CallingConvention::ThisCall)
);
}
#[test]
fn test_call_lowering_lower_vectorcall_call_x64() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::VectorCall);
let args = vec![make_test_arg(8)];
let ret = Some(make_test_ret(8, false));
let site = cl.lower_vectorcall_call("vec_func", &args, ret.as_ref());
assert_eq!(site.callee, "vec_func");
assert_eq!(
site.call_conv_override,
Some(X86CallingConvention::VectorCall)
);
}
#[test]
fn test_call_lowering_lower_regcall_call() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_RegCall);
let args = vec![make_test_arg(8), make_test_arg(8)];
let ret = Some(make_test_ret(8, false));
let site = cl.lower_regcall_call("regcall_func", &args, ret.as_ref());
assert_eq!(site.callee, "regcall_func");
assert_eq!(
site.call_conv_override,
Some(X86CallingConvention::X86_RegCall)
);
}
#[test]
fn test_call_lowering_lower_sret() {
let mut cl = X86CallLowering::default();
let sret = cl.lower_sret(32, 8);
assert!(sret.is_sret);
assert_eq!(sret.size, 32);
assert_eq!(sret.alignment, 8);
assert!(!sret.regs.is_empty());
}
#[test]
fn test_call_lowering_lower_byval() {
let mut cl = X86CallLowering::default();
let byval = cl.lower_byval(16, 8);
assert!(byval.is_byval);
assert_eq!(byval.size, 16);
assert_eq!(byval.alignment, 8);
}
#[test]
fn test_call_lowering_lower_inalloca() {
let mut cl = X86CallLowering::default();
let inalloca = cl.lower_inalloca(32, 16);
assert!(inalloca.is_inalloca);
assert_eq!(inalloca.size, 32);
assert_eq!(inalloca.alignment, 16);
}
#[test]
fn test_call_lowering_lower_varargs_sysv64() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let info = cl.lower_varargs();
assert_eq!(info.va_list_size, 24);
assert_eq!(info.num_gpr_save, 6);
assert_eq!(info.num_xmm_save, 8);
assert!(info.reg_save_area_size > 0);
}
#[test]
fn test_call_lowering_lower_varargs_win64() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let info = cl.lower_varargs();
assert_eq!(info.num_gpr_save, 4);
assert_eq!(info.num_xmm_save, 4);
}
#[test]
fn test_call_lowering_lower_varargs_32bit() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let info = cl.lower_varargs();
assert_eq!(info.num_gpr_save, 0);
assert_eq!(info.num_xmm_save, 0);
}
#[test]
fn test_call_lowering_is_tail_call_eligible() {
let cl = X86CallLowering::default();
assert!(cl.is_tail_call_eligible(
X86CallingConvention::X86_64_SysV,
X86CallingConvention::X86_64_SysV,
true,
));
assert!(!cl.is_tail_call_eligible(
X86CallingConvention::X86_64_SysV,
X86CallingConvention::Win64,
false,
));
}
#[test]
fn test_arg_lowering_new() {
let al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(al.int_regs_used, 0);
assert_eq!(al.sse_regs_used, 0);
assert_eq!(al.stack_offset, 0);
}
#[test]
fn test_arg_lowering_lower_arguments_sysv64() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let args = vec![make_test_arg(8), make_test_arg(8)];
let lowered = al.lower_arguments(&args);
assert_eq!(lowered.len(), 2);
}
#[test]
fn test_arg_lowering_lower_arguments_win64() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let args = vec![
make_test_arg(8),
make_test_arg(8),
make_test_arg(8),
make_test_arg(8),
make_test_arg(8),
];
let lowered = al.lower_arguments(&args);
assert_eq!(lowered.len(), 5);
}
#[test]
fn test_arg_lowering_assign_sysv64_integer() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let arg = make_test_arg(8);
let result = al.assign_sysv64_arg(&arg);
assert!(!result.regs.is_empty());
assert_eq!(result.regs[0].reg_num, 7);
}
#[test]
fn test_arg_lowering_assign_win64_integer() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let arg = make_test_arg(8);
let result = al.assign_win64_arg(&arg);
assert!(!result.regs.is_empty());
assert_eq!(result.regs[0].reg_num, 1);
}
#[test]
fn test_arg_lowering_assign_fastcall_32() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::Fast);
let arg = make_test_arg(4);
let result = al.assign_fastcall_arg(&arg);
assert!(!result.regs.is_empty());
assert_eq!(result.regs[0].reg_num, 1); }
#[test]
fn test_arg_lowering_assign_thiscall() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::ThisCall);
let arg = make_test_arg(4);
let result = al.assign_thiscall_arg(&arg);
assert!(!result.regs.is_empty());
assert_eq!(result.regs[0].reg_num, 1); }
#[test]
fn test_arg_lowering_reset() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let _ = al.lower_arguments(&[make_test_arg(8)]);
assert!(al.int_regs_used > 0);
al.reset();
assert_eq!(al.int_regs_used, 0);
assert_eq!(al.sse_regs_used, 0);
assert_eq!(al.stack_offset, 0);
}
#[test]
fn test_arg_lowering_get_int_arg_regs_sysv64() {
let al = X86ArgLowering::default();
let regs = al.get_int_arg_regs();
assert_eq!(regs.len(), 6);
assert_eq!(regs[0], 7); assert_eq!(regs[1], 6); }
#[test]
fn test_arg_lowering_get_int_arg_regs_win64() {
let al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let regs = al.get_int_arg_regs();
assert_eq!(regs.len(), 4);
assert_eq!(regs[0], 1); }
#[test]
fn test_arg_lowering_get_sse_arg_regs() {
let al = X86ArgLowering::default();
let regs = al.get_sse_arg_regs();
assert_eq!(regs.len(), 8);
}
#[test]
fn test_arg_lowering_remaining_regs() {
let mut al = X86ArgLowering::default();
assert_eq!(al.remaining_int_regs(), 6);
let _ = al.lower_arguments(&[make_test_arg(8)]);
assert_eq!(al.remaining_int_regs(), 5);
}
#[test]
fn test_ret_lowering_new() {
let rl = X86RetLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(rl.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_ret_lowering_lower_return_sysv64_int_8() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(8, false);
assert!(!ret.uses_sret);
assert_eq!(ret.regs.len(), 1);
assert_eq!(ret.regs[0].reg_num, 0); assert_eq!(ret.regs[0].reg_class, X86LoweringRegClass::GPR64);
}
#[test]
fn test_ret_lowering_lower_return_sysv64_int_16() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(16, false);
assert!(!ret.uses_sret);
assert!(ret.is_multi_reg);
assert_eq!(ret.regs.len(), 2);
assert_eq!(ret.regs[0].reg_num, 0); assert_eq!(ret.regs[1].reg_num, 2); }
#[test]
fn test_ret_lowering_lower_return_sysv64_int_large() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(32, false);
assert!(ret.uses_sret);
assert!(ret.sret_reg.is_some());
}
#[test]
fn test_ret_lowering_lower_return_sysv64_fp_8() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(8, true);
assert!(!ret.uses_sret);
assert_eq!(ret.regs.len(), 1);
assert_eq!(ret.regs[0].reg_class, X86LoweringRegClass::XMM);
assert_eq!(ret.regs[0].reg_num, 0); }
#[test]
fn test_ret_lowering_lower_return_win64() {
let rl = X86RetLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let ret = rl.lower_return_by_size(8, false);
assert!(!ret.uses_sret);
assert_eq!(ret.regs.len(), 1);
assert_eq!(ret.regs[0].reg_num, 0); }
#[test]
fn test_ret_lowering_lower_return_32bit() {
let rl = X86RetLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let ret = rl.lower_return_by_size(4, false);
assert!(!ret.uses_sret);
assert_eq!(ret.regs.len(), 1);
assert_eq!(ret.regs[0].reg_class, X86LoweringRegClass::GPR32);
assert_eq!(ret.regs[0].reg_num, 0); }
#[test]
fn test_ret_lowering_needs_sret() {
let rl = X86RetLowering::default();
assert!(!rl.needs_sret(8, false));
assert!(!rl.needs_sret(16, false));
assert!(rl.needs_sret(32, false));
assert!(!rl.needs_sret(8, true));
assert!(!rl.needs_sret(16, true));
assert!(rl.needs_sret(32, true));
}
#[test]
fn test_ret_lowering_needs_sret_win64() {
let rl = X86RetLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert!(rl.needs_sret(9, false));
}
#[test]
fn test_frame_lowering_new() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(fl.has_red_zone);
assert_eq!(fl.push_size, 8);
assert_eq!(fl.fp_reg, 5); assert_eq!(fl.sp_reg, 4); assert!(!fl.uses_seh);
}
#[test]
fn test_frame_lowering_new_win64() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert!(!fl.has_red_zone);
assert!(fl.uses_seh);
}
#[test]
fn test_frame_lowering_new_32bit() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
assert_eq!(fl.push_size, 4);
}
#[test]
fn test_frame_lowering_compute_frame_layout() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let layout = fl.compute_frame_layout();
assert!(layout.has_red_zone);
assert_eq!(layout.red_zone_size, 128);
assert!(layout.has_frame_pointer);
}
#[test]
fn test_frame_lowering_emit_prologue() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_prologue(32);
assert!(!instrs.is_empty());
assert!(instrs.len() >= 2);
}
#[test]
fn test_frame_lowering_emit_epilogue() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_epilogue(32);
assert!(!instrs.is_empty());
let last = instrs.last().unwrap();
assert_eq!(last.opcode, MOpcode::G_RETURN);
assert!(last.flags.is_terminator);
}
#[test]
fn test_frame_lowering_can_use_red_zone() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(fl.can_use_red_zone(true, 64));
assert!(!fl.can_use_red_zone(false, 0));
assert!(!fl.can_use_red_zone(true, 256));
}
#[test]
fn test_frame_lowering_can_use_red_zone_win64() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert!(!fl.can_use_red_zone(true, 0));
}
#[test]
fn test_frame_lowering_can_eliminate_frame_pointer() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(fl.can_eliminate_frame_pointer(false, false, 0));
assert!(!fl.can_eliminate_frame_pointer(true, false, 0));
}
#[test]
fn test_frame_lowering_should_eliminate_frame_pointer() {
let fl = X86FrameLoweringEngine::default();
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O0, false, false, 0);
assert!(!info.eliminated);
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O2, false, false, 0);
assert!(info.eliminated);
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O2, true, false, 0);
assert!(!info.eliminated);
}
#[test]
fn test_frame_lowering_emit_seh_prologue() {
let mut fl =
X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let instrs = fl.emit_seh_prologue(64);
assert!(!instrs.is_empty());
}
#[test]
fn test_frame_lowering_needs_stack_probe() {
let fl = X86FrameLoweringEngine::default();
assert_eq!(fl.needs_stack_probe(0), StackProbeKind::None);
assert_eq!(fl.needs_stack_probe(512), StackProbeKind::None);
assert_eq!(fl.needs_stack_probe(5000), StackProbeKind::Inline);
}
#[test]
fn test_frame_lowering_emit_prologue_32() {
let mut fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let instrs = fl.emit_prologue_32(16);
assert!(!instrs.is_empty());
}
#[test]
fn test_frame_lowering_emit_epilogue_32() {
let mut fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let instrs = fl.emit_epilogue_32(16);
assert!(!instrs.is_empty());
}
#[test]
fn test_callee_saved_new() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
assert_eq!(cs.callee_saved_regs.len(), 6);
assert!(cs.callee_saved_regs.contains(&3)); assert!(cs.callee_saved_regs.contains(&5)); assert!(cs.callee_saved_regs.contains(&12)); }
#[test]
fn test_callee_saved_new_32bit() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_32);
assert_eq!(cs.callee_saved_regs.len(), 4);
assert!(cs.callee_saved_regs.contains(&3)); assert!(cs.callee_saved_regs.contains(&5)); }
#[test]
fn test_callee_saved_compute_spills() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let frame = X86FrameLayout::default();
let info = cs.compute_spills(&frame);
assert!(!info.regs_to_save.is_empty());
assert!(info.total_size > 0);
}
#[test]
fn test_callee_saved_assign_spill_slots() {
let mut cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let slots = cs.assign_spill_slots(&[3, 12, 13], 16);
assert_eq!(slots.len(), 3);
assert_eq!(slots[0].vreg, 3);
assert_eq!(slots[0].offset, 16);
assert_eq!(slots[0].size, 8);
}
#[test]
fn test_callee_saved_is_shrink_wrap_eligible() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let info = cs.is_shrink_wrap_eligible(false, &[3, 12]);
assert!(!info.profitable);
let info = cs.is_shrink_wrap_eligible(true, &[3, 12]);
assert!(info.profitable);
}
#[test]
fn test_callee_saved_get_callee_saved_gprs() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let gprs = cs.get_callee_saved_gprs();
assert!(!gprs.is_empty());
}
#[test]
fn test_callee_saved_with_calling_convention() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64)
.with_calling_convention(X86CallingConvention::Win64);
assert!(!cs.callee_saved_regs.contains(&15));
}
#[test]
fn test_callee_saved_get_callee_saved_xmms_win64() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let xmms = cs.get_callee_saved_xmms_win64();
assert!(!xmms.is_empty());
assert!(xmms.contains(&38));
assert!(xmms.contains(&47));
}
#[test]
fn test_shadow_stack_new() {
let ss = X86ShadowStack::new(X86TargetVariant::X86_64);
assert!(!ss.enabled);
assert!(ss.insert_endbr);
}
#[test]
fn test_shadow_stack_lower_disabled() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64);
ss.lower();
assert!(ss.prologue.is_empty());
assert!(ss.epilogue.is_empty());
}
#[test]
fn test_shadow_stack_lower_enabled() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64).with_shadow_stack(true);
ss.lower();
assert!(!ss.prologue.is_empty());
assert!(!ss.epilogue.is_empty());
}
#[test]
fn test_shadow_stack_with_endbr() {
let ss = X86ShadowStack::default().with_endbr(true);
assert!(ss.insert_endbr);
}
#[test]
fn test_shadow_stack_without_endbr() {
let ss = X86ShadowStack::default().with_endbr(false);
assert!(!ss.insert_endbr);
}
#[test]
fn test_shadow_stack_with_shadow_stack() {
let ss = X86ShadowStack::default().with_shadow_stack(true);
assert!(ss.enabled);
}
#[test]
fn test_eh_pad_lowering_new() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
assert!(!ehl.uses_eh);
assert!(ehl.personality_fn.is_none());
}
#[test]
fn test_eh_pad_lowering_lower_landing_pads() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 0,
catch_types: vec![1, 2],
cleanup: true,
filter_types: vec![],
}];
ehl.lower_landing_pads(&pads);
assert!(ehl.uses_eh);
assert_eq!(ehl.landing_pads.len(), 1);
assert_eq!(ehl.landing_pads[0].catches.len(), 2);
assert!(ehl.landing_pads[0].has_cleanup);
}
#[test]
fn test_eh_pad_lowering_generate_dispatch() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 0,
catch_types: vec![1],
cleanup: false,
filter_types: vec![],
}];
ehl.lower_landing_pads(&pads);
let instrs = ehl.generate_dispatch(0);
assert!(!instrs.is_empty());
}
#[test]
fn test_eh_pad_lowering_generate_resume() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let instrs = ehl.generate_resume();
assert_eq!(instrs.len(), 1);
assert_eq!(instrs[0].opcode, MOpcode::G_RESUME);
}
#[test]
fn test_eh_pad_lowering_generate_cleanup() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let instrs = ehl.generate_cleanup();
assert_eq!(instrs.len(), 1);
assert_eq!(instrs[0].opcode, MOpcode::G_CLEANUPRET);
}
#[test]
fn test_eh_pad_lowering_with_personality() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64)
.with_personality("__gxx_personality_v0");
assert_eq!(ehl.personality_fn, Some("__gxx_personality_v0".to_string()));
}
#[test]
fn test_eh_pad_lowering_empty_landing_pads() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
ehl.lower_landing_pads(&[]);
assert!(!ehl.uses_eh);
assert!(ehl.landing_pads.is_empty());
}
#[test]
fn test_dwarf_cfi_new() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
assert!(cfi.enabled);
assert!(cfi.cie.is_cie);
assert_eq!(cfi.cie.version, 1);
assert_eq!(cfi.cie.return_address_register, DWARF_RETURN_ADDR);
assert_eq!(cfi.cie.initial_cfa_register, DWARF_RSP);
}
#[test]
fn test_dwarf_cfi_emit_cfi() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_cfi();
assert_eq!(cfi.current_cfa_register, DWARF_RSP);
assert!(cfi.directive_count > 0);
}
#[test]
fn test_dwarf_cfi_emit_def_cfa() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_def_cfa(DWARF_RBP, 16);
assert_eq!(cfi.current_cfa_register, DWARF_RBP);
assert_eq!(cfi.current_cfa_offset, 16);
}
#[test]
fn test_dwarf_cfi_emit_def_cfa_offset() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_def_cfa_offset(32);
assert_eq!(cfi.current_cfa_offset, 32);
}
#[test]
fn test_dwarf_cfi_emit_def_cfa_register() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_def_cfa_register(DWARF_RBP);
assert_eq!(cfi.current_cfa_register, DWARF_RBP);
}
#[test]
fn test_dwarf_cfi_emit_offset() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_offset(DWARF_RBX, -16);
assert!(!cfi.register_save_locations.is_empty());
}
#[test]
fn test_dwarf_cfi_emit_same_value() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_same_value(DWARF_R12);
assert!(!cfi.register_save_locations.is_empty());
}
#[test]
fn test_dwarf_cfi_emit_restore() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_offset(DWARF_RBX, -16);
cfi.emit_restore(DWARF_RBX);
assert!(cfi
.register_save_locations
.iter()
.all(|(r, _, _)| *r != DWARF_RBX));
}
#[test]
fn test_dwarf_cfi_begin_end_fde() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.begin_fde(0x1000, 256);
assert_eq!(cfi.fdes.len(), 1);
assert_eq!(cfi.fdes[0].pc_begin, Some(0x1000));
assert_eq!(cfi.fdes[0].pc_range, Some(256));
cfi.end_fde();
assert!(!cfi.fdes[0].instructions.is_empty());
}
#[test]
fn test_dwarf_cfi_emit_cie_bytes() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
let bytes = cfi.emit_cie_bytes();
assert!(!bytes.is_empty());
}
#[test]
fn test_dwarf_cfi_emit_fde_bytes() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.begin_fde(0x2000, 512);
cfi.end_fde();
let bytes = cfi.emit_fde_bytes(0);
assert!(!bytes.is_empty());
}
#[test]
fn test_dwarf_cfi_directive_count() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
assert_eq!(cfi.directive_count(), 0);
cfi.emit_cfi();
assert!(cfi.directive_count() > 0);
}
#[test]
fn test_dwarf_cfi_is_enabled() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
assert!(cfi.is_enabled());
let cfi = cfi.with_cfi(false);
assert!(!cfi.is_enabled());
}
#[test]
fn test_dwarf_cfi_disabled_emit_cfi() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64).with_cfi(false);
cfi.emit_cfi();
assert_eq!(cfi.directive_count(), 0);
}
#[test]
fn test_align_to() {
assert_eq!(align_to(0, 16), 0);
assert_eq!(align_to(1, 16), 16);
assert_eq!(align_to(15, 16), 16);
assert_eq!(align_to(16, 16), 16);
assert_eq!(align_to(17, 16), 32);
assert_eq!(align_to(0, 0), 0);
assert_eq!(align_to(5, 0), 5);
assert_eq!(align_to(7, 8), 8);
}
#[test]
fn test_fits_i8() {
assert!(fits_i8(0));
assert!(fits_i8(127));
assert!(fits_i8(-128));
assert!(!fits_i8(128));
assert!(!fits_i8(-129));
assert!(fits_i8(-1));
}
#[test]
fn test_fits_i32() {
assert!(fits_i32(0));
assert!(fits_i32(2147483647));
assert!(fits_i32(-2147483648));
assert!(!fits_i32(2147483648));
assert!(!fits_i32(-2147483649));
}
#[test]
fn test_is_imm8() {
assert!(is_imm8(0));
assert!(is_imm8(-1));
assert!(is_imm8(127));
assert!(!is_imm8(256));
}
#[test]
fn test_is_imm32() {
assert!(is_imm32(0));
assert!(is_imm32(-1));
assert!(is_imm32(2147483647));
assert!(!is_imm32(3000000000i64));
}
#[test]
fn test_type_size_bytes() {
assert_eq!(type_size_bytes(0), 0); assert_eq!(type_size_bytes(1), 1); assert_eq!(type_size_bytes(8), 1); assert_eq!(type_size_bytes(16), 2); assert_eq!(type_size_bytes(32), 4); assert_eq!(type_size_bytes(64), 8); assert_eq!(type_size_bytes(33), 4); assert_eq!(type_size_bytes(65), 8); assert_eq!(type_size_bytes(128), 16); assert_eq!(type_size_bytes(999), 8); }
#[test]
fn test_encode_uleb128() {
let bytes = encode_uleb128(0);
assert_eq!(bytes, vec![0]);
let bytes = encode_uleb128(127);
assert_eq!(bytes, vec![127]);
let bytes = encode_uleb128(128);
assert_eq!(bytes, vec![0x80, 0x01]);
let bytes = encode_uleb128(16383);
assert_eq!(bytes, vec![0xff, 0x7f]);
}
#[test]
fn test_encode_sleb128() {
let bytes = encode_sleb128(0);
assert_eq!(bytes, vec![0]);
let bytes = encode_sleb128(1);
assert_eq!(bytes, vec![1]);
let bytes = encode_sleb128(-1);
assert_eq!(bytes, vec![0x7f]);
let bytes = encode_sleb128(127);
assert_eq!(bytes, vec![0xff, 0x00]);
let bytes = encode_sleb128(-128);
assert_eq!(bytes, vec![0x80, 0x7f]);
}
#[test]
fn test_create_x86_64_sysv_lowering() {
let lowering = create_x86_64_sysv_lowering();
assert_eq!(lowering.target_variant, X86TargetVariant::X86_64);
assert_eq!(
lowering.calling_convention,
X86CallingConvention::X86_64_SysV
);
}
#[test]
fn test_create_x86_64_win64_lowering() {
let lowering = create_x86_64_win64_lowering();
assert_eq!(lowering.calling_convention, X86CallingConvention::Win64);
}
#[test]
fn test_create_x86_32_cdecl_lowering() {
let lowering = create_x86_32_cdecl_lowering();
assert_eq!(lowering.target_variant, X86TargetVariant::X86_32);
assert_eq!(lowering.calling_convention, X86CallingConvention::C);
}
#[test]
fn test_create_ir_transformer() {
let t = create_x86_ir_transformer(X86TargetVariant::X86_64);
assert!(t.supports_expansion("llvm.memcpy"));
}
#[test]
fn test_create_call_lowering() {
let cl =
create_x86_call_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(cl.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_create_arg_lowering() {
let al =
create_x86_arg_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(al.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_create_ret_lowering() {
let rl =
create_x86_ret_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(rl.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_create_frame_lowering() {
let fl =
create_x86_frame_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert!(fl.has_red_zone);
}
#[test]
fn test_create_callee_saved() {
let cs = create_x86_callee_saved(X86TargetVariant::X86_64);
assert!(!cs.callee_saved_regs.is_empty());
}
#[test]
fn test_create_shadow_stack() {
let ss = create_x86_shadow_stack(X86TargetVariant::X86_64);
assert!(!ss.enabled);
}
#[test]
fn test_create_eh_pad_lowering() {
let ehl = create_x86_eh_pad_lowering(X86TargetVariant::X86_64);
assert!(!ehl.uses_eh);
}
#[test]
fn test_create_dwarf_cfi() {
let cfi = create_x86_dwarf_cfi(X86TargetVariant::X86_64);
assert!(cfi.enabled);
}
#[test]
fn test_full_lowering_pipeline_sysv64() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let args = vec![make_test_arg(8), make_test_arg(8), make_test_arg(4)];
let ret = Some(make_test_ret(8, false));
let result = lowering.lower_function("full_test", &args, ret.as_ref());
assert!(result.function_state.is_some());
assert!(result.ret_lowered.is_some());
assert!(!result.ret_lowered.as_ref().unwrap().regs.is_empty());
assert_eq!(result.args_lowered.len(), 3);
assert!(result.frame_computed.alignment > 0);
}
#[test]
fn test_full_lowering_pipeline_win64() {
let mut lowering = X86Lowering::new_x86_64_win64();
let args = vec![
make_test_arg(8),
make_test_arg(8),
make_test_arg(8),
make_test_arg(8),
make_test_arg(8), ];
let ret = Some(make_test_ret(8, false));
let result = lowering.lower_function("win64_test", &args, ret.as_ref());
assert_eq!(result.args_lowered.len(), 5);
let stack_args: Vec<_> = result
.args_lowered
.iter()
.filter(|a| a.stack_offset.is_some())
.collect();
assert!(!stack_args.is_empty());
}
#[test]
fn test_full_lowering_pipeline_32bit() {
let mut lowering = X86Lowering::new_x86_32_cdecl();
let args = vec![make_test_arg(4), make_test_arg(4)];
let ret = Some(make_test_ret(4, false));
let result = lowering.lower_function("cdecl_test", &args, ret.as_ref());
assert_eq!(result.args_lowered.len(), 2);
for arg in &result.args_lowered {
assert!(arg.stack_offset.is_some() || !arg.regs.is_empty());
}
}
#[test]
fn test_call_lowering_full_pipeline_sysv64() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let args = vec![make_test_arg(8), make_test_arg(4), make_test_arg(8)];
let ret = Some(make_test_ret(8, false));
let site = cl.lower_sysv64_call("full_pipeline_callee", &args, ret.as_ref());
assert!(!site.arg_assignments.is_empty());
assert!(site.ret_assignment.is_some());
}
#[test]
fn test_frame_lowering_prologue_epilogue_roundtrip() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let prologue = fl.emit_prologue(64);
let epilogue = fl.emit_epilogue(64);
assert!(!prologue.is_empty());
assert!(!epilogue.is_empty());
let last = epilogue.last().unwrap();
assert!(last.flags.is_return);
}
#[test]
fn test_shadow_stack_with_cet_enabled_lowering() {
let mut lowering = X86Lowering::new_x86_64_sysv().with_cet(true);
lowering.begin_function("cet_test");
lowering.shadow_stack.lower();
assert!(lowering.shadow_stack.prologue.len() >= 2); assert!(!lowering.shadow_stack.epilogue.is_empty());
}
#[test]
fn test_ir_transformer_then_call_lowering() {
let mut transformer = X86IRTransformer::new(X86TargetVariant::X86_64);
let transform_result = transformer.transform();
assert!(transform_result.instructions_transformed > 0);
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let site = cl.lower_sysv64_call(
"post_transform",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(site.callee, "post_transform");
}
#[test]
fn test_eh_pad_lowering_with_frame_lowering() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
ehl.lower_landing_pads(&[LandingPadDescriptor {
block_id: 0,
catch_types: vec![1],
cleanup: true,
filter_types: vec![],
}]);
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let prologue = fl.emit_prologue(48);
assert!(!prologue.is_empty());
assert!(ehl.uses_eh);
}
#[test]
fn test_dwarf_cfi_with_frame_lowering() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let _ = fl.emit_prologue(32);
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_cfi();
cfi.begin_fde(0x4000, 128);
cfi.end_fde();
assert!(cfi.fdes.len() == 1);
assert!(cfi.directive_count() > 0);
}
#[test]
fn test_arg_lowering_many_args() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let args: Vec<X86LoweringArg> = (0..10).map(|_| make_test_arg(8)).collect();
let lowered = al.lower_arguments(&args);
assert_eq!(lowered.len(), 10);
let stack_count = lowered.iter().filter(|a| a.stack_offset.is_some()).count();
assert!(stack_count >= 4);
}
#[test]
fn test_call_lowering_zero_args() {
let mut cl = X86CallLowering::default();
let site = cl.lower_sysv64_call("no_args_func", &[], None);
assert_eq!(site.callee, "no_args_func");
assert!(site.arg_assignments.is_empty());
assert!(site.ret_assignment.is_none());
assert_eq!(site.arg_stack_size, 0);
}
#[test]
fn test_frame_lowering_zero_frame() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let prologue = fl.emit_prologue(0);
assert!(!prologue.is_empty()); }
#[test]
fn test_ret_lowering_void_equivalent() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(0, false);
assert!(ret.regs.is_empty());
assert!(!ret.uses_sret);
}
#[test]
fn test_lowering_lower_frame_only() {
let mut lowering = X86Lowering::default();
let layout = lowering.lower_frame_only();
assert!(layout.has_red_zone);
}
#[test]
fn test_lowering_lower_calls_only() {
let mut lowering = X86Lowering::default();
let count = lowering.lower_calls_only();
assert_eq!(count, 0); assert_eq!(lowering.stats.calls_lowered, 0);
}
#[test]
fn test_ir_transformer_transform_empty_function_ok() {
let mut transformer = X86IRTransformer::default()
.with_expand_mem(false)
.with_expand_math(false);
let result = transformer.transform();
assert_eq!(result.intrinsics_expanded, 0);
}
#[test]
fn test_lowering_with_avx512_implies_avx() {
let lowering = X86Lowering::default().with_avx512(true);
assert!(lowering.has_avx512);
assert!(lowering.has_avx);
}
#[test]
fn test_lowering_mode_default() {
assert_eq!(X86LoweringMode::default(), X86LoweringMode::SelectionDAG);
}
#[test]
fn test_opt_level_default() {
assert_eq!(X86OptLevel::default(), X86OptLevel::O0);
}
#[test]
fn test_opt_level_ordering() {
assert!(X86OptLevel::O0 < X86OptLevel::O3);
assert!(X86OptLevel::O1 > X86OptLevel::O0);
}
#[test]
fn test_lowering_result_default() {
let result = X86LoweringResult::default();
assert!(result.function_state.is_none());
assert!(result.args_lowered.is_empty());
assert!(result.ret_lowered.is_none());
assert_eq!(result.stats.functions_lowered, 0);
}
#[test]
fn test_lowering_stats_default() {
let stats = X86LoweringStats::default();
assert_eq!(stats.functions_lowered, 0);
assert_eq!(stats.calls_lowered, 0);
assert_eq!(stats.args_lowered, 0);
}
#[test]
fn test_call_site_info_default() {
let site = X86CallSiteInfo::default();
assert!(site.callee.is_empty());
assert!(!site.tail_call_eligible);
assert!(!site.uses_sret);
assert!(!site.has_byval);
assert!(!site.has_inalloca);
}
#[test]
fn test_arg_lowering_default() {
let al = X86ArgLowering::default();
assert_eq!(al.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_ret_lowering_default() {
let rl = X86RetLowering::default();
assert_eq!(rl.calling_convention, X86CallingConvention::X86_64_SysV);
}
#[test]
fn test_frame_lowering_engine_default() {
let fl = X86FrameLoweringEngine::default();
assert!(fl.has_red_zone);
}
#[test]
fn test_callee_saved_default() {
let cs = X86CalleeSaved::default();
assert!(!cs.callee_saved_regs.is_empty());
}
#[test]
fn test_shadow_stack_default() {
let ss = X86ShadowStack::default();
assert!(!ss.enabled);
assert!(ss.insert_endbr);
}
#[test]
fn test_eh_pad_lowering_default() {
let ehl = X86EHPadLowering::default();
assert!(!ehl.uses_eh);
}
#[test]
fn test_dwarf_cfi_default() {
let cfi = X86DwarfCFI::default();
assert!(cfi.enabled);
}
#[test]
fn test_multi_abi_full_pipeline_sysv64() {
let mut lowering = X86Lowering::new_x86_64_sysv();
lowering.begin_function("multi_sysv");
let _ = lowering.lower_function(
"multi_sysv",
&[make_test_arg(8), make_test_arg(16)],
Some(&make_test_ret(8, true)),
);
assert_eq!(lowering.stats.functions_lowered, 1);
}
#[test]
fn test_multi_abi_full_pipeline_win64() {
let mut lowering = X86Lowering::new_x86_64_win64();
let _ = lowering.lower_function(
"multi_win64",
&[make_test_arg(8), make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(lowering.stats.functions_lowered, 1);
}
#[test]
fn test_multi_abi_full_pipeline_cdecl() {
let mut lowering = X86Lowering::new_x86_32_cdecl();
let _ = lowering.lower_function(
"multi_cdecl",
&[make_test_arg(4), make_test_arg(4)],
Some(&make_test_ret(4, false)),
);
assert_eq!(lowering.stats.functions_lowered, 1);
}
#[test]
fn test_multi_abi_fastcall_lowering() {
let mut lowering = X86Lowering::new(X86TargetVariant::X86_32)
.with_calling_convention(X86CallingConvention::Fast);
let _ = lowering.lower_function(
"multi_fastcall",
&[make_test_arg(4)],
Some(&make_test_ret(4, false)),
);
assert_eq!(lowering.stats.functions_lowered, 1);
}
#[test]
fn test_multi_abi_regcall_lowering() {
let mut lowering = X86Lowering::new(X86TargetVariant::X86_64)
.with_calling_convention(X86CallingConvention::X86_RegCall);
let _ = lowering.lower_function(
"multi_regcall",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(lowering.stats.functions_lowered, 1);
}
#[test]
fn test_concurrent_lowering_operations() {
let mut lowering1 = X86Lowering::new_x86_64_sysv();
let mut lowering2 = X86Lowering::new_x86_64_win64();
lowering1.begin_function("func1");
lowering2.begin_function("func2");
let result1 =
lowering1.lower_function("func1", &[make_test_arg(8)], Some(&make_test_ret(8, false)));
let result2 =
lowering2.lower_function("func2", &[make_test_arg(8)], Some(&make_test_ret(8, false)));
assert_eq!(result1.stats.functions_lowered, 1);
assert_eq!(result2.stats.functions_lowered, 1);
}
#[test]
fn test_large_argument_list() {
let mut al = X86ArgLowering::default();
let args: Vec<X86LoweringArg> = (0..50).map(|_| make_test_arg(8)).collect();
let lowered = al.lower_arguments(&args);
assert_eq!(lowered.len(), 50);
let stack_count = lowered.iter().filter(|a| a.stack_offset.is_some()).count();
assert!(stack_count >= 44); }
#[test]
fn test_huge_frame_size() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let prologue = fl.emit_prologue(1_000_000);
assert!(!prologue.is_empty());
assert_eq!(fl.needs_stack_probe(1_000_000), StackProbeKind::Inline);
}
#[test]
fn test_byval_and_sret_together() {
let mut cl = X86CallLowering::default();
let sret = cl.lower_sret(24, 8);
let byval = cl.lower_byval(16, 8);
assert!(sret.is_sret);
assert!(byval.is_byval);
let site = cl.lower_sysv64_call(
"sret_byval_func",
&[sret, byval, make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert!(site.uses_sret);
assert!(site.has_byval);
}
#[test]
fn test_ir_transformer_memcpy_expansion() {
let mut transformer = X86IRTransformer::new(X86TargetVariant::X86_64)
.with_expand_mem(true)
.with_expand_math(false);
let result = transformer.transform();
assert!(result.instructions_transformed >= 10);
}
#[test]
fn test_ir_transformer_math_expansion_x86_64() {
let mut transformer = X86IRTransformer::new(X86TargetVariant::X86_64)
.with_expand_mem(false)
.with_expand_math(true);
let result = transformer.transform();
assert!(result.intrinsics_expanded > 0);
}
#[test]
fn test_ir_transformer_target_intrinsics_x86_64() {
let mut transformer = X86IRTransformer::new(X86TargetVariant::X86_64)
.with_expand_mem(false)
.with_expand_math(false);
let result = transformer.transform();
assert!(result.target_intrinsics_expanded > 0);
}
#[test]
fn test_ir_transformer_all_math_intrinsics_supported() {
let transformer = X86IRTransformer::default();
let math_intrinsics = [
"llvm.sqrt.f32",
"llvm.sqrt.f64",
"llvm.sin.f32",
"llvm.sin.f64",
"llvm.cos.f32",
"llvm.cos.f64",
"llvm.pow.f32",
"llvm.pow.f64",
"llvm.fma.f32",
"llvm.fma.f64",
"llvm.ceil.f32",
"llvm.ceil.f64",
"llvm.floor.f32",
"llvm.floor.f64",
"llvm.trunc.f32",
"llvm.trunc.f64",
"llvm.rint.f32",
"llvm.rint.f64",
"llvm.nearbyint.f32",
"llvm.nearbyint.f64",
"llvm.round.f32",
"llvm.round.f64",
];
for intr in &math_intrinsics {
assert!(transformer.supports_expansion(intr), "Missing: {}", intr);
}
}
#[test]
fn test_ir_transformer_all_bit_intrinsics_supported() {
let transformer = X86IRTransformer::default();
let bit_intrinsics = [
"llvm.ctpop.i32",
"llvm.ctpop.i64",
"llvm.ctlz.i32",
"llvm.ctlz.i64",
"llvm.cttz.i32",
"llvm.cttz.i64",
"llvm.bswap.i16",
"llvm.bswap.i32",
"llvm.bswap.i64",
"llvm.fshl.i32",
"llvm.fshr.i32",
"llvm.abs.i32",
"llvm.abs.i64",
"llvm.smax.i32",
"llvm.smin.i32",
"llvm.umax.i32",
"llvm.umin.i32",
];
for intr in &bit_intrinsics {
assert!(transformer.supports_expansion(intr), "Missing: {}", intr);
}
}
#[test]
fn test_ir_transformer_all_sse_intrinsics_supported() {
let transformer = X86IRTransformer::default();
let sse_intrinsics = [
"llvm.x86.sse2.add.sd",
"llvm.x86.sse2.sub.sd",
"llvm.x86.sse2.mul.sd",
"llvm.x86.sse2.div.sd",
"llvm.x86.sse.add.ss",
"llvm.x86.sse.sub.ss",
"llvm.x86.sse.mul.ss",
"llvm.x86.sse.div.ss",
];
for intr in &sse_intrinsics {
assert!(transformer.supports_expansion(intr), "Missing: {}", intr);
}
}
#[test]
fn test_ir_transformer_all_avx_intrinsics_supported() {
let transformer = X86IRTransformer::default();
let avx_intrinsics = [
"llvm.x86.avx.add.pd.256",
"llvm.x86.avx.mul.pd.256",
"llvm.x86.avx2.gather.d.d",
"llvm.x86.avx2.gather.q.d",
"llvm.x86.fma.vfmadd.pd",
];
for intr in &avx_intrinsics {
assert!(transformer.supports_expansion(intr), "Missing: {}", intr);
}
}
#[test]
fn test_ir_transformer_all_crypto_intrinsics_supported() {
let transformer = X86IRTransformer::default();
let crypto_intrinsics = [
"llvm.x86.aesni.aesenc",
"llvm.x86.aesni.aesenclast",
"llvm.x86.aesni.aesdec",
"llvm.x86.aesni.aesdeclast",
"llvm.x86.sha1rnds4",
"llvm.x86.sha256rnds2",
];
for intr in &crypto_intrinsics {
assert!(transformer.supports_expansion(intr), "Missing: {}", intr);
}
}
#[test]
fn test_ir_transformer_32bit_supported_intrinsics() {
let transformer = X86IRTransformer::new(X86TargetVariant::X86_32);
assert!(transformer.supports_expansion("llvm.memcpy"));
assert!(transformer.supports_expansion("llvm.sqrt.f32"));
assert!(transformer.supports_expansion("llvm.x86.sse.add.ss"));
}
#[test]
fn test_ir_transformer_32bit_memcpy_count() {
let mut transformer = X86IRTransformer::new(X86TargetVariant::X86_32);
let result = transformer.transform();
assert!(result.instructions_transformed <= 15); }
#[test]
fn test_call_lowering_sysv64_all_args_in_regs() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let args: Vec<X86LoweringArg> = (0..6).map(|_| make_test_arg(8)).collect();
let site = cl.lower_sysv64_call("all_regs", &args, None);
let reg_count = site
.arg_assignments
.iter()
.filter(|a| !a.regs.is_empty())
.count();
assert_eq!(reg_count, 6);
assert_eq!(site.arg_stack_size, 0);
}
#[test]
fn test_call_lowering_sysv64_mixed_reg_stack() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let args: Vec<X86LoweringArg> = (0..10).map(|_| make_test_arg(8)).collect();
let site = cl.lower_sysv64_call("mixed", &args, None);
let reg_count = site
.arg_assignments
.iter()
.filter(|a| !a.regs.is_empty())
.count();
assert_eq!(reg_count, 6);
assert!(site.arg_stack_size > 0);
}
#[test]
fn test_call_lowering_win64_all_args_in_regs() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let args: Vec<X86LoweringArg> = (0..4).map(|_| make_test_arg(8)).collect();
let site = cl.lower_win64_call("win64_all_regs", &args, None);
let reg_count = site
.arg_assignments
.iter()
.filter(|a| !a.regs.is_empty())
.count();
assert_eq!(reg_count, 4);
}
#[test]
fn test_call_lowering_win64_shadow_space_reserved() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let site = cl.lower_win64_call("shadow", &[], None);
assert_eq!(site.arg_stack_size, WIN64_SHADOW_SPACE);
}
#[test]
fn test_call_lowering_cdecl_args_on_stack() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let args: Vec<X86LoweringArg> = (0..5).map(|_| make_test_arg(4)).collect();
let site = cl.lower_cdecl_call("cdecl", &args, None);
let stack_count = site
.arg_assignments
.iter()
.filter(|a| a.stack_offset.is_some())
.count();
assert_eq!(stack_count, 5);
}
#[test]
fn test_call_lowering_fastcall_first_two_in_regs() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::Fast);
let args: Vec<X86LoweringArg> = (0..4).map(|_| make_test_arg(4)).collect();
let site = cl.lower_fastcall_call("fastcall", &args, None);
let reg_count = site
.arg_assignments
.iter()
.filter(|a| !a.regs.is_empty())
.count();
assert_eq!(reg_count, 2);
assert!(site.arg_stack_size > 0);
}
#[test]
fn test_call_lowering_thiscall_this_in_ecx() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::ThisCall);
let args: Vec<X86LoweringArg> = (0..3).map(|_| make_test_arg(4)).collect();
let site = cl.lower_thiscall_call("thiscall", &args, None);
assert!(!site.arg_assignments.is_empty());
let first = &site.arg_assignments[0];
assert!(!first.regs.is_empty());
assert_eq!(first.regs[0].reg_num, 1);
assert_eq!(first.regs[0].reg_class, X86LoweringRegClass::GPR32);
}
#[test]
fn test_call_lowering_vectorcall_x64_simd_regs() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::VectorCall);
let vec_arg = X86LoweringArg {
size: 16,
alignment: 16,
type_id: Some(64), ..Default::default()
};
let site = cl.lower_vectorcall_call("vec", &[vec_arg], None);
assert!(!site.arg_assignments.is_empty());
}
#[test]
fn test_call_lowering_vectorcall_32_simd_regs() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::VectorCall);
let vec_arg = X86LoweringArg {
size: 16,
alignment: 16,
type_id: Some(64),
..Default::default()
};
let site = cl.lower_vectorcall_call("vec32", &[vec_arg], None);
assert!(!site.arg_assignments.is_empty());
}
#[test]
fn test_call_lowering_regcall_x64_max_regs() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_RegCall);
let args: Vec<X86LoweringArg> = (0..11).map(|_| make_test_arg(8)).collect();
let site = cl.lower_regcall_call("regcall", &args, None);
let reg_count = site
.arg_assignments
.iter()
.filter(|a| !a.regs.is_empty())
.count();
assert_eq!(reg_count, 11);
}
#[test]
fn test_call_lowering_regcall_32_max_regs() {
let mut cl =
X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::X86_RegCall);
let args: Vec<X86LoweringArg> = (0..5).map(|_| make_test_arg(4)).collect();
let site = cl.lower_regcall_call("regcall32", &args, None);
let reg_count = site
.arg_assignments
.iter()
.filter(|a| !a.regs.is_empty())
.count();
assert_eq!(reg_count, 5);
}
#[test]
fn test_call_lowering_stdcall_callee_cleans() {
let mut cl = X86CallLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let args = vec![make_test_arg(4)];
let site = cl.lower_stdcall_call("stdcallee", &args, None);
assert_eq!(site.call_conv_override, Some(X86CallingConvention::StdCall));
}
#[test]
fn test_call_lowering_tail_call_blocked_by_byval() {
let mut cl = X86CallLowering::default();
let byval_arg = X86LoweringArg {
is_byval: true,
size: 16,
alignment: 8,
..Default::default()
};
let site = cl.lower_call(
"tail_target",
&[byval_arg],
None,
CallLoweringType::TailCall,
);
assert!(!site.tail_call_eligible);
assert!(site.tail_call_blocker.is_some());
}
#[test]
fn test_call_lowering_tail_call_blocked_by_inalloca() {
let mut cl = X86CallLowering::default();
let inalloca_arg = X86LoweringArg {
is_inalloca: true,
size: 32,
alignment: 16,
..Default::default()
};
let site = cl.lower_call(
"tail_target",
&[inalloca_arg],
None,
CallLoweringType::TailCall,
);
assert!(!site.tail_call_eligible);
}
#[test]
fn test_call_lowering_musttail_fails_if_not_eligible() {
let mut cl = X86CallLowering::default();
cl.enable_tail_calls = false;
let site = cl.lower_call(
"musttail_target",
&[make_test_arg(8)],
None,
CallLoweringType::MustTail,
);
assert!(!site.tail_call_eligible);
}
#[test]
fn test_call_lowering_retain_site_info() {
let mut cl = X86CallLowering::default();
let _ = cl.lower_sysv64_call("func_a", &[make_test_arg(8)], None);
let _ = cl.lower_sysv64_call("func_b", &[make_test_arg(8)], None);
assert_eq!(cl.call_sites.len(), 2);
assert_eq!(cl.call_sites[0].callee, "func_a");
assert_eq!(cl.call_sites[1].callee, "func_b");
}
#[test]
fn test_arg_lowering_sysv64_struct_split_across_regs() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let struct_arg = X86LoweringArg {
size: 16,
alignment: 8,
..Default::default()
};
let result = al.lower_single_arg(&struct_arg);
assert_eq!(result.regs.len(), 2);
}
#[test]
fn test_arg_lowering_sysv64_byval_indirect() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let byval_arg = X86LoweringArg {
is_byval: true,
size: 64,
alignment: 8,
..Default::default()
};
let result = al.lower_single_arg(&byval_arg);
assert!(!result.regs.is_empty());
}
#[test]
fn test_arg_lowering_sysv64_sret_first_arg() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let sret_arg = X86LoweringArg {
is_sret: true,
size: 32,
alignment: 8,
..Default::default()
};
let result = al.lower_single_arg(&sret_arg);
assert!(!result.regs.is_empty());
assert_eq!(result.regs[0].reg_num, 7); }
#[test]
fn test_arg_lowering_win64_large_struct_indirect() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let large_arg = X86LoweringArg {
size: 32,
alignment: 8,
is_byval: true,
..Default::default()
};
let result = al.lower_single_arg(&large_arg);
assert!(!result.regs.is_empty() || result.stack_offset.is_some());
}
#[test]
fn test_arg_lowering_sse_arg_regs_count() {
let al = X86ArgLowering::default();
let regs = al.get_sse_arg_regs();
assert_eq!(regs.len(), 8);
assert_eq!(regs[0], 32); assert_eq!(regs[7], 39); }
#[test]
fn test_arg_lowering_promote_floats_32bit() {
let al = X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
assert!(!al.promote_floats);
}
#[test]
fn test_arg_lowering_get_int_arg_regs_fastcall() {
let al = X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::Fast);
let regs = al.get_int_arg_regs();
assert_eq!(regs.len(), 2);
assert_eq!(regs[0], 1); assert_eq!(regs[1], 2); }
#[test]
fn test_arg_lowering_get_int_arg_regs_empty_for_cdecl() {
let al = X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let regs = al.get_int_arg_regs();
assert!(regs.is_empty());
}
#[test]
fn test_arg_lowering_remaining_regs_counts_down() {
let mut al = X86ArgLowering::default();
assert_eq!(al.remaining_int_regs(), 6);
assert_eq!(al.remaining_sse_regs(), 8);
let _ = al.lower_single_arg(&make_test_arg(8));
assert_eq!(al.remaining_int_regs(), 5);
assert_eq!(al.remaining_sse_regs(), 8); }
#[test]
fn test_arg_lowering_exhaust_regs_32bit() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
assert_eq!(al.remaining_int_regs(), 0);
assert_eq!(al.remaining_sse_regs(), 0);
}
#[test]
fn test_arg_lowering_vectorcall_sse_first() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::VectorCall);
let vec_arg = X86LoweringArg {
size: 16,
alignment: 16,
..Default::default()
};
let result = al.assign_vectorcall_arg(&vec_arg);
assert!(!result.regs.is_empty());
assert_eq!(result.regs[0].reg_class, X86LoweringRegClass::XMM);
}
#[test]
fn test_ret_lowering_sysv64_hfa_return() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(16, true);
assert!(!ret.uses_sret);
assert!(ret.is_multi_reg);
assert_eq!(ret.regs.len(), 2);
assert_eq!(ret.regs[0].reg_class, X86LoweringRegClass::XMM);
assert_eq!(ret.regs[1].reg_class, X86LoweringRegClass::XMM);
}
#[test]
fn test_ret_lowering_sysv64_i128_return() {
let rl = X86RetLowering::default();
let ret = rl.lower_return_by_size(16, false);
assert!(!ret.uses_sret);
assert_eq!(ret.regs.len(), 2);
assert_eq!(ret.regs[0].reg_num, 0); assert_eq!(ret.regs[1].reg_num, 2); }
#[test]
fn test_ret_lowering_win64_aggregate_sret() {
let rl = X86RetLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert!(rl.needs_sret(16, false));
assert!(rl.needs_sret(9, false));
let ret = rl.lower_return_by_size(16, false);
assert!(ret.uses_sret);
}
#[test]
fn test_ret_lowering_32bit_i64_return_edx_eax() {
let rl = X86RetLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let ret = rl.lower_return_by_size(8, false);
assert!(!ret.uses_sret);
assert!(ret.is_multi_reg);
assert_eq!(ret.regs.len(), 2);
assert_eq!(ret.regs[0].reg_class, X86LoweringRegClass::GPR32);
assert_eq!(ret.regs[0].reg_num, 0); assert_eq!(ret.regs[1].reg_num, 2); }
#[test]
fn test_ret_lowering_32bit_fp_return_xmm0() {
let rl = X86RetLowering::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let ret = rl.lower_return_by_size(4, true);
assert!(!ret.uses_sret);
assert_eq!(ret.regs.len(), 1);
assert_eq!(ret.regs[0].reg_class, X86LoweringRegClass::XMM);
}
#[test]
fn test_ret_lowering_get_int_return_regs_small() {
let rl = X86RetLowering::default();
let regs = rl.get_int_return_regs(4);
assert_eq!(regs.len(), 1);
assert_eq!(regs[0].reg_num, 0); }
#[test]
fn test_ret_lowering_get_int_return_regs_large() {
let rl = X86RetLowering::default();
let regs = rl.get_int_return_regs(32);
assert!(regs.is_empty()); }
#[test]
fn test_ret_lowering_get_fp_return_regs_double() {
let rl = X86RetLowering::default();
let regs = rl.get_fp_return_regs(8);
assert_eq!(regs.len(), 1);
assert_eq!(regs[0].reg_class, X86LoweringRegClass::XMM);
}
#[test]
fn test_ret_lowering_get_fp_return_regs_fp128() {
let rl = X86RetLowering::default();
let regs = rl.get_fp_return_regs(16);
assert_eq!(regs.len(), 2);
}
#[test]
fn test_ret_lowering_lower_none() {
let rl = X86RetLowering::default();
let result = rl.lower_return(None);
assert!(result.is_none());
}
#[test]
fn test_frame_lowering_prologue_has_push_mov_sub() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_prologue(64);
assert!(instrs.len() >= 3);
assert_eq!(instrs[0].opcode, MOpcode::G_STORE);
assert_eq!(instrs[1].opcode, MOpcode::G_BITCAST);
assert_eq!(instrs[2].opcode, MOpcode::G_SUB);
}
#[test]
fn test_frame_lowering_epilogue_has_mov_pop_ret() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_epilogue(64);
assert!(instrs.len() >= 3);
let last = instrs.last().unwrap();
assert_eq!(last.opcode, MOpcode::G_RETURN);
}
#[test]
fn test_frame_lowering_win64_seh_prologue() {
let mut fl =
X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let instrs = fl.emit_seh_prologue(128);
assert!(!instrs.is_empty());
}
#[test]
fn test_frame_lowering_seh_prologue_not_emitted_for_sysv() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_seh_prologue(64);
assert!(instrs.is_empty()); }
#[test]
fn test_frame_lowering_red_zone_at_boundary() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(fl.can_use_red_zone(true, 128));
assert!(!fl.can_use_red_zone(true, 129));
}
#[test]
fn test_frame_lowering_32bit_no_red_zone() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
assert!(!fl.can_use_red_zone(true, 0));
}
#[test]
fn test_frame_lowering_eliminate_fp_var_sized() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(!fl.can_eliminate_frame_pointer(true, false, 0));
}
#[test]
fn test_frame_lowering_eliminate_fp_win64_seh() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let can_elim = fl.can_eliminate_frame_pointer(false, true, 64);
assert!(!can_elim);
}
#[test]
fn test_frame_lowering_stack_probe_thresholds() {
let fl = X86FrameLoweringEngine::default();
assert_eq!(fl.needs_stack_probe(100), StackProbeKind::None);
assert_eq!(fl.needs_stack_probe(4095), StackProbeKind::None);
assert_eq!(fl.needs_stack_probe(4096), StackProbeKind::None); assert_eq!(fl.needs_stack_probe(5000), StackProbeKind::Inline);
}
#[test]
fn test_frame_lowering_win64_chkstk_probe() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert_eq!(fl.needs_stack_probe(10000), StackProbeKind::ChkStk);
}
#[test]
fn test_frame_lowering_emit_chkstk_probe() {
let mut fl =
X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let instrs = fl.emit_stack_probe(10000);
assert!(!instrs.is_empty());
}
#[test]
fn test_frame_lowering_emit_inline_probe() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_stack_probe(5000);
assert!(!instrs.is_empty());
}
#[test]
fn test_frame_lowering_no_probe_small_frame() {
let mut fl = X86FrameLoweringEngine::default();
let instrs = fl.emit_stack_probe(256);
assert!(instrs.is_empty());
}
#[test]
fn test_frame_lowering_realignment_prologue() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_realignment_prologue(32);
assert!(!instrs.is_empty());
assert!(instrs.len() >= 3);
}
#[test]
fn test_frame_lowering_prologue_32_has_push_ebp() {
let mut fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let instrs = fl.emit_prologue_32(32);
assert!(instrs.len() >= 3);
assert_eq!(instrs[0].opcode, MOpcode::G_STORE); }
#[test]
fn test_frame_lowering_epilogue_32_has_pop_ebp_ret() {
let mut fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let instrs = fl.emit_epilogue_32(32);
assert!(instrs.len() >= 3);
}
#[test]
fn test_frame_lowering_should_eliminate_fp_o0() {
let fl = X86FrameLoweringEngine::default();
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O0, false, false, 0);
assert!(!info.eliminated);
assert_eq!(info.reason, FpElimReason::DebugInfo);
}
#[test]
fn test_frame_lowering_should_eliminate_fp_o2() {
let fl = X86FrameLoweringEngine::default();
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O2, false, false, 0);
assert!(info.eliminated);
}
#[test]
fn test_frame_lowering_should_not_eliminate_fp_var_sized() {
let fl = X86FrameLoweringEngine::default();
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O2, true, false, 0);
assert!(!info.eliminated);
assert_eq!(info.reason, FpElimReason::VarSizedObjects);
}
#[test]
fn test_frame_lowering_should_not_eliminate_fp_win64_calls() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let info = fl.should_eliminate_frame_pointer(X86OptLevel::O2, false, true, 64);
assert!(!info.eliminated);
}
#[test]
fn test_callee_saved_sysv64_full_set() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let regs = cs.get_callee_saved_gprs();
assert_eq!(regs.len(), 6);
assert!(regs.contains(&3)); assert!(regs.contains(&5)); assert!(regs.contains(&12)); assert!(regs.contains(&13)); assert!(regs.contains(&14)); assert!(regs.contains(&15)); }
#[test]
fn test_callee_saved_win64_r15_not_saved() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64)
.with_calling_convention(X86CallingConvention::Win64);
let regs = cs.get_callee_saved_gprs();
assert!(!regs.contains(&15)); assert!(regs.contains(&6)); assert!(regs.contains(&7)); }
#[test]
fn test_callee_saved_32bit_full_set() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_32);
let regs = cs.get_callee_saved_gprs();
assert_eq!(regs.len(), 4);
assert!(regs.contains(&3)); assert!(regs.contains(&5)); assert!(regs.contains(&6)); assert!(regs.contains(&7)); }
#[test]
fn test_callee_saved_assign_spill_slots_with_offset() {
let mut cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let slots = cs.assign_spill_slots(&[3, 12, 13, 14, 15], 32);
assert_eq!(slots.len(), 5);
assert_eq!(slots[0].offset, 32);
assert_eq!(slots[1].offset, 40);
assert_eq!(slots[2].offset, 48);
assert_eq!(slots[3].offset, 56);
assert_eq!(slots[4].offset, 64);
}
#[test]
fn test_callee_saved_assign_spill_slots_32bit() {
let mut cs = X86CalleeSaved::new(X86TargetVariant::X86_32);
let slots = cs.assign_spill_slots(&[3, 5, 6, 7], 0);
assert_eq!(slots.len(), 4);
assert_eq!(slots[0].size, 4); assert_eq!(slots[0].offset, 0);
assert_eq!(slots[1].offset, 4);
assert_eq!(slots[2].offset, 8);
assert_eq!(slots[3].offset, 12);
}
#[test]
fn test_callee_saved_shrink_wrap_no_calls_not_profitable() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let info = cs.is_shrink_wrap_eligible(false, &[3, 12]);
assert!(!info.profitable);
}
#[test]
fn test_callee_saved_shrink_wrap_no_shrinkable_regs() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let info = cs.is_shrink_wrap_eligible(true, &[5]); assert!(!info.profitable);
}
#[test]
fn test_callee_saved_shrink_wrap_with_shrinkable_regs() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let info = cs.is_shrink_wrap_eligible(true, &[3, 12]); assert!(info.profitable);
assert!(info.save_block.is_some());
assert!(info.restore_block.is_some());
assert!(!info.shrinkable_regs.is_empty());
}
#[test]
fn test_callee_saved_xmm_win64_count() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let xmms = cs.get_callee_saved_xmms_win64();
assert_eq!(xmms.len(), 10); }
#[test]
fn test_callee_saved_xmm_sysv64_count() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let xmms = cs.get_callee_saved_xmms_sysv64();
assert!(xmms.is_empty()); }
#[test]
fn test_callee_saved_compute_spills_has_size() {
let cs = X86CalleeSaved::new(X86TargetVariant::X86_64);
let frame = X86FrameLayout {
callee_saved_start: 16,
..Default::default()
};
let info = cs.compute_spills(&frame);
assert!(info.total_size >= 16); }
#[test]
fn test_shadow_stack_prologue_count_enabled() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64)
.with_shadow_stack(true)
.with_endbr(true);
ss.lower();
assert!(ss.prologue.len() >= 3);
}
#[test]
fn test_shadow_stack_epilogue_has_restore() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64).with_shadow_stack(true);
ss.lower();
assert!(!ss.epilogue.is_empty());
}
#[test]
fn test_shadow_stack_disabled_no_output() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64).with_shadow_stack(false);
ss.lower();
assert!(ss.prologue.is_empty());
assert!(ss.epilogue.is_empty());
}
#[test]
fn test_shadow_stack_endbr_disabled() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64)
.with_shadow_stack(true)
.with_endbr(false);
ss.lower();
assert_eq!(ss.prologue.len(), 2);
}
#[test]
fn test_shadow_stack_32bit_disabled_by_default() {
let ss = X86ShadowStack::new(X86TargetVariant::X86_32);
assert!(!ss.enabled);
}
#[test]
fn test_eh_pad_multiple_landing_pads() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![
LandingPadDescriptor {
block_id: 1,
catch_types: vec![1],
cleanup: false,
filter_types: vec![],
},
LandingPadDescriptor {
block_id: 2,
catch_types: vec![2, 3],
cleanup: true,
filter_types: vec![],
},
];
ehl.lower_landing_pads(&pads);
assert_eq!(ehl.landing_pads.len(), 2);
assert!(ehl.uses_eh);
}
#[test]
fn test_eh_pad_dispatch_with_multiple_catches() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 5,
catch_types: vec![1, 2, 3],
cleanup: false,
filter_types: vec![],
}];
ehl.lower_landing_pads(&pads);
let instrs = ehl.generate_dispatch(0);
assert!(instrs.len() >= 4); }
#[test]
fn test_eh_pad_dispatch_with_cleanup() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 0,
catch_types: vec![],
cleanup: true,
filter_types: vec![],
}];
ehl.lower_landing_pads(&pads);
let instrs = ehl.generate_dispatch(0);
assert!(!instrs.is_empty());
}
#[test]
fn test_eh_pad_dispatch_with_filters() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 3,
catch_types: vec![],
cleanup: false,
filter_types: vec![1, 2],
}];
ehl.lower_landing_pads(&pads);
let instrs = ehl.generate_dispatch(0);
assert!(!instrs.is_empty());
}
#[test]
fn test_eh_pad_invalid_pad_index() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let instrs = ehl.generate_dispatch(999);
assert!(instrs.is_empty());
}
#[test]
fn test_eh_pad_with_personality_sets_field() {
let ehl =
X86EHPadLowering::new(X86TargetVariant::X86_64).with_personality("__CxxFrameHandler3");
assert_eq!(ehl.personality_fn, Some("__CxxFrameHandler3".to_string()));
}
#[test]
fn test_eh_pad_resume_instruction_flags() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let instrs = ehl.generate_resume();
assert_eq!(instrs.len(), 1);
assert!(instrs[0].flags.is_terminator);
assert!(instrs[0].flags.is_barrier);
}
#[test]
fn test_eh_pad_cleanup_instruction_flags() {
let ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let instrs = ehl.generate_cleanup();
assert_eq!(instrs.len(), 1);
assert!(instrs[0].flags.is_terminator);
}
#[test]
fn test_dwarf_cfi_32bit_different_ra() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_32);
assert_eq!(cfi.cie.return_address_register, 8);
assert_eq!(cfi.cie.data_alignment_factor, -4);
}
#[test]
fn test_dwarf_cfi_32bit_initial_cfa_offset() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_32);
assert_eq!(cfi.cie.initial_cfa_offset, 4); }
#[test]
fn test_dwarf_cfi_prologue_cfa_sequence() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_cfi();
assert_eq!(cfi.current_cfa_register, DWARF_RSP);
assert_eq!(cfi.current_cfa_offset, 8);
cfi.emit_def_cfa(DWARF_RSP, 16);
assert_eq!(cfi.current_cfa_offset, 16);
cfi.emit_offset(DWARF_RBP, -16);
cfi.emit_def_cfa_register(DWARF_RBP);
assert_eq!(cfi.current_cfa_register, DWARF_RBP);
}
#[test]
fn test_dwarf_cfi_multiple_register_saves() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_offset(DWARF_RBX, -24);
cfi.emit_offset(DWARF_R12, -32);
cfi.emit_offset(DWARF_R13, -40);
assert_eq!(cfi.register_save_locations.len(), 3);
}
#[test]
fn test_dwarf_cfi_restore_clears_save_location() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_offset(DWARF_RBX, -24);
cfi.emit_offset(DWARF_R12, -32);
cfi.emit_restore(DWARF_RBX);
assert_eq!(cfi.register_save_locations.len(), 1);
assert_eq!(cfi.register_save_locations[0].0, DWARF_R12);
}
#[test]
fn test_dwarf_cfi_same_value_added_to_locations() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_same_value(DWARF_RAX);
assert_eq!(cfi.register_save_locations.len(), 1);
}
#[test]
fn test_dwarf_cfi_begin_fde_resets_directive_count() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_cfi();
assert!(cfi.directive_count() > 0);
cfi.begin_fde(0x1000, 256);
assert_eq!(cfi.directive_count(), 0);
}
#[test]
fn test_dwarf_cfi_multiple_fdes() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.begin_fde(0x1000, 128);
cfi.end_fde();
cfi.begin_fde(0x2000, 256);
cfi.end_fde();
cfi.begin_fde(0x3000, 512);
cfi.end_fde();
assert_eq!(cfi.fdes.len(), 3);
}
#[test]
fn test_dwarf_cfi_fde_inherits_cie_parameters() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.begin_fde(0x4000, 64);
let fde = &cfi.fdes[0];
assert_eq!(fde.code_alignment_factor, cfi.cie.code_alignment_factor);
assert_eq!(fde.data_alignment_factor, cfi.cie.data_alignment_factor);
assert_eq!(fde.return_address_register, cfi.cie.return_address_register);
}
#[test]
fn test_dwarf_cfi_cie_bytes_non_empty() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
let bytes = cfi.emit_cie_bytes();
assert!(!bytes.is_empty());
assert!(bytes.len() >= 8);
}
#[test]
fn test_dwarf_cfi_fde_bytes_non_empty() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.begin_fde(0x5000, 1024);
cfi.end_fde();
let bytes = cfi.emit_fde_bytes(0);
assert!(!bytes.is_empty());
}
#[test]
fn test_dwarf_cfi_fde_invalid_index() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
let bytes = cfi.emit_fde_bytes(999);
assert!(bytes.is_empty());
}
#[test]
fn test_lowering_result_default_values() {
let result = X86LoweringResult::default();
assert!(result.function_state.is_none());
assert!(result.args_lowered.is_empty());
assert!(result.ret_lowered.is_none());
assert_eq!(result.stats.functions_lowered, 0);
assert_eq!(result.stats.calls_lowered, 0);
assert_eq!(result.stats.args_lowered, 0);
}
#[test]
fn test_lowering_result_with_stats() {
let mut lowering = X86Lowering::default();
lowering.stats.functions_lowered = 5;
lowering.stats.calls_lowered = 42;
lowering.stats.args_lowered = 128;
let result = lowering.lower_function("stats_test", &[], None);
assert_eq!(result.stats.functions_lowered, 6);
assert_eq!(result.stats.args_lowered, 128);
}
#[test]
fn test_function_state_default() {
let state = X86FunctionLoweringState::default();
assert!(state.name.is_empty());
assert!(!state.is_leaf);
assert!(!state.has_var_sized_objects);
assert!(!state.has_calls);
assert!(!state.needs_frame_pointer);
assert_eq!(state.num_args, 0);
assert_eq!(state.frame_size, 0);
assert!(state.callee_saved_regs.is_empty());
assert!(state.arg_assignments.is_empty());
assert!(state.ret_assignment.is_none());
}
#[test]
fn test_varargs_info_structure() {
let info = VarArgsLoweringInfo {
gp_offset_offset: 0,
fp_offset_offset: 4,
overflow_arg_area_offset: 8,
reg_save_area_offset: 16,
va_list_size: 24,
num_gpr_save: 6,
num_xmm_save: 8,
reg_save_area_size: 176,
};
assert_eq!(info.va_list_size, 24);
assert!(info.reg_save_area_size > 0);
}
#[test]
fn test_call_lowering_type_variants() {
assert_eq!(
CallLoweringType::Standard as i32,
CallLoweringType::Standard as i32
);
assert_ne!(
CallLoweringType::Standard as i32,
CallLoweringType::TailCall as i32
);
}
#[test]
fn test_full_pipeline_with_eh() {
let mut lowering = X86Lowering::new_x86_64_sysv();
lowering.begin_function("eh_func");
if let Some(ref mut state) = lowering.function_state {
state.uses_eh = true;
state.landing_pads.push(LandingPadDescriptor {
block_id: 0,
catch_types: vec![1],
cleanup: true,
filter_types: vec![],
});
state.personality_fn = Some("__gxx_personality_v0".to_string());
}
let result = lowering.lower_function(
"eh_func",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(result.stats.landing_pads_lowered, 1);
}
#[test]
fn test_full_pipeline_with_cet() {
let mut lowering = X86Lowering::new_x86_64_sysv().with_cet(true);
let result = lowering.lower_function(
"cet_func",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(result.stats.shadow_stack_instructions, 2);
}
#[test]
fn test_full_pipeline_with_cfi() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let result = lowering.lower_function(
"cfi_func",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert!(result.stats.cfi_directives_emitted > 0);
}
#[test]
fn test_full_pipeline_leaf_function_red_zone() {
let mut lowering = X86Lowering::new_x86_64_sysv();
lowering.begin_function("leaf");
if let Some(ref mut state) = lowering.function_state {
state.is_leaf = true;
state.has_calls = false;
}
let result = lowering.lower_function("leaf", &[], None);
assert!(result.frame_computed.has_red_zone);
}
#[test]
fn test_full_pipeline_non_leaf_no_red_zone() {
let mut lowering = X86Lowering::new_x86_64_sysv();
lowering.begin_function("nonleaf");
if let Some(ref mut state) = lowering.function_state {
state.is_leaf = false;
state.has_calls = true;
}
let result = lowering.lower_function("nonleaf", &[], None);
assert!(result.frame_computed.has_red_zone);
}
#[test]
fn test_full_pipeline_large_arg_list() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let args: Vec<X86LoweringArg> = (0..20).map(|_| make_test_arg(8)).collect();
let result = lowering.lower_function("large_args", &args, None);
assert_eq!(result.args_lowered.len(), 20);
}
#[test]
fn test_full_pipeline_function_with_hfa_return() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let hfa_ret = X86LoweringRet {
regs: vec![X86LoweringReg::xmm(0), X86LoweringReg::xmm(1)],
size: 16,
is_multi_reg: true,
..Default::default()
};
let result = lowering.lower_function("hfa_func", &[make_test_arg(8)], Some(&hfa_ret));
assert!(result.ret_lowered.is_some());
assert_eq!(result.ret_lowered.as_ref().unwrap().regs.len(), 2);
}
#[test]
fn test_full_pipeline_all_stats_accumulate() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let _ = lowering.lower_function("f1", &[make_test_arg(8)], Some(&make_test_ret(8, false)));
let _ = lowering.lower_function(
"f2",
&[make_test_arg(8), make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
let _ = lowering.lower_function(
"f3",
&[make_test_arg(8), make_test_arg(8), make_test_arg(8)],
None,
);
assert_eq!(lowering.stats.functions_lowered, 3);
assert_eq!(lowering.stats.args_lowered, 6);
assert_eq!(lowering.stats.returns_lowered, 2);
}
#[test]
fn test_full_pipeline_byval_argument_passthrough() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let byval_arg = X86LoweringArg {
is_byval: true,
size: 64,
alignment: 8,
..Default::default()
};
let result = lowering.lower_function("byval_func", &[byval_arg], None);
assert_eq!(result.args_lowered.len(), 1);
assert!(result.args_lowered[0].is_byval);
}
#[test]
fn test_full_pipeline_sret_argument() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let sret_arg = X86LoweringArg {
is_sret: true,
size: 32,
alignment: 8,
..Default::default()
};
let result = lowering.lower_function("sret_func", &[sret_arg], None);
assert_eq!(result.args_lowered.len(), 1);
assert!(result.args_lowered[0].is_sret);
}
#[test]
fn test_lowering_stats_clone() {
let mut stats = X86LoweringStats::default();
stats.functions_lowered = 10;
stats.calls_lowered = 100;
let cloned = stats.clone();
assert_eq!(cloned.functions_lowered, 10);
assert_eq!(cloned.calls_lowered, 100);
}
#[test]
fn test_lowering_large_frame_size_probe() {
let mut lowering = X86Lowering::new_x86_64_sysv();
lowering.begin_function("large_frame");
if let Some(ref mut state) = lowering.function_state {
state.frame_size = 100_000;
}
let frame = lowering.lower_frame_only();
assert!(frame.alignment > 0);
}
#[test]
fn test_lowering_zero_arg_function() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let result = lowering.lower_function("zero_args", &[], None);
assert!(result.args_lowered.is_empty());
assert!(result.ret_lowered.is_none());
}
#[test]
fn test_lowering_zero_ret_function() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let void_ret = X86LoweringRet {
size: 0,
..Default::default()
};
let result = lowering.lower_function("void_ret", &[make_test_arg(8)], Some(&void_ret));
assert!(result.ret_lowered.is_some());
assert_eq!(result.ret_lowered.as_ref().unwrap().size, 0);
}
#[test]
fn test_all_target_variants_create_lowering() {
let x64 = X86Lowering::new(X86TargetVariant::X86_64);
let x32 = X86Lowering::new(X86TargetVariant::X86_32);
let x32_abi = X86Lowering::new(X86TargetVariant::X32);
let x16 = X86Lowering::new(X86TargetVariant::X86_16);
assert!(x64.use_red_zone);
assert!(!x32.use_red_zone);
assert!(x32_abi.use_red_zone); assert!(!x16.use_red_zone);
}
#[test]
fn test_target_variant_pointer_width_matches_register_width() {
assert_eq!(X86TargetVariant::X86_64.pointer_width(), 64);
assert_eq!(X86TargetVariant::X86_64.register_width(), 64);
assert_eq!(X86TargetVariant::X86_32.pointer_width(), 32);
assert_eq!(X86TargetVariant::X86_32.register_width(), 32);
assert_eq!(X86TargetVariant::X32.pointer_width(), 32);
assert_eq!(X86TargetVariant::X32.register_width(), 64);
}
#[test]
fn test_all_cc_have_lowering_config() {
let ccs = vec![
X86CallingConvention::C,
X86CallingConvention::Fast,
X86CallingConvention::StdCall,
X86CallingConvention::ThisCall,
X86CallingConvention::VectorCall,
X86CallingConvention::X86_64_SysV,
X86CallingConvention::Win64,
X86CallingConvention::X86_RegCall,
];
for cc in &ccs {
let lowering = X86Lowering::new(X86TargetVariant::X86_64).with_calling_convention(*cc);
assert_eq!(lowering.calling_convention, *cc);
}
}
#[test]
fn test_lowering_pipeline_stages_in_order() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let transform = lowering.ir_transformer.transform();
assert!(transform.instructions_transformed > 0);
let calls = lowering.lower_calls_only();
assert_eq!(calls, 0);
let frame = lowering.lower_frame_only();
assert!(frame.has_red_zone);
}
#[test]
fn test_opt_level_os_oz_variants() {
assert_eq!(X86OptLevel::Os as u8, 4);
assert_eq!(X86OptLevel::Oz as u8, 5);
assert!(X86OptLevel::Os > X86OptLevel::O3);
assert!(X86OptLevel::Oz > X86OptLevel::Os);
}
#[test]
fn test_lowering_mode_variants() {
assert_eq!(
X86LoweringMode::SelectionDAG as u8,
X86LoweringMode::SelectionDAG as u8
);
assert_ne!(
X86LoweringMode::SelectionDAG as u8,
X86LoweringMode::GlobalISel as u8
);
}
#[test]
fn test_x86_conv_to_lowering_cc_sysv64() {
let lowering = X86Lowering::new_x86_64_sysv();
assert_eq!(
lowering.calling_convention,
X86CallingConvention::X86_64_SysV
);
assert!(lowering.frame_lowering.has_red_zone);
}
#[test]
fn test_x86_conv_to_lowering_cc_win64() {
let lowering = X86Lowering::new_x86_64_win64();
assert_eq!(lowering.calling_convention, X86CallingConvention::Win64);
assert!(lowering.frame_lowering.uses_seh);
}
#[test]
fn test_machine_operand_mem_full() {
let op = X86MachineOperand::Mem {
base: Some(VReg(1)),
index: Some(VReg(2)),
scale: 4,
displacement: 16,
size: 8,
};
match op {
X86MachineOperand::Mem {
base,
index,
scale,
displacement,
size,
} => {
assert_eq!(base, Some(VReg(1)));
assert_eq!(index, Some(VReg(2)));
assert_eq!(scale, 4);
assert_eq!(displacement, 16);
assert_eq!(size, 8);
}
_ => panic!("Expected Mem variant"),
}
}
#[test]
fn test_machine_operand_global() {
let op = X86MachineOperand::Global("my_symbol".to_string());
match op {
X86MachineOperand::Global(s) => assert_eq!(s, "my_symbol"),
_ => panic!("Expected Global variant"),
}
}
#[test]
fn test_x86_mi_flags_call() {
let flags = X86MIFlags {
is_call: true,
has_side_effects: true,
is_barrier: true,
..Default::default()
};
assert!(flags.is_call);
assert!(flags.has_side_effects);
assert!(flags.is_barrier);
}
#[test]
fn test_x86_mi_flags_load_store() {
let flags = X86MIFlags {
may_load: true,
may_store: true,
..Default::default()
};
assert!(flags.may_load);
assert!(flags.may_store);
}
#[test]
fn test_x86_mi_flags_tail_call() {
let flags = X86MIFlags {
is_call: true,
is_tail_call: true,
is_terminator: true,
..Default::default()
};
assert!(flags.is_call);
assert!(flags.is_tail_call);
assert!(flags.is_terminator);
}
#[test]
fn test_x86_machine_instr_full_construction() {
let instr = X86MachineInstr {
opcode: MOpcode::G_ADD,
defs: vec![VReg(1)],
uses: vec![X86MachineOperand::vreg(VReg(2)), X86MachineOperand::imm(42)],
flags: X86MIFlags::default(),
};
assert_eq!(instr.opcode, MOpcode::G_ADD);
assert_eq!(instr.defs.len(), 1);
assert_eq!(instr.uses.len(), 2);
}
#[test]
fn test_uleb128_encoded_values() {
assert_eq!(encode_uleb128(0), vec![0x00]);
assert_eq!(encode_uleb128(1), vec![0x01]);
assert_eq!(encode_uleb128(127), vec![0x7f]);
assert_eq!(encode_uleb128(128), vec![0x80, 0x01]);
assert_eq!(encode_uleb128(255), vec![0xff, 0x01]);
assert_eq!(encode_uleb128(256), vec![0x80, 0x02]);
assert_eq!(encode_uleb128(16383), vec![0xff, 0x7f]);
assert_eq!(encode_uleb128(16384), vec![0x80, 0x80, 0x01]);
}
#[test]
fn test_sleb128_encoded_values() {
assert_eq!(encode_sleb128(0), vec![0x00]);
assert_eq!(encode_sleb128(1), vec![0x01]);
assert_eq!(encode_sleb128(-1), vec![0x7f]);
assert_eq!(encode_sleb128(63), vec![0x3f]);
assert_eq!(encode_sleb128(-64), vec![0x40]);
assert_eq!(encode_sleb128(64), vec![0xc0, 0x00]);
assert_eq!(encode_sleb128(-65), vec![0xbf, 0x7f]);
assert_eq!(encode_sleb128(127), vec![0xff, 0x00]);
assert_eq!(encode_sleb128(-128), vec![0x80, 0x7f]);
}
#[test]
fn test_uleb128_roundtrip_small() {
for value in 0u64..1000 {
let encoded = encode_uleb128(value);
assert!(!encoded.is_empty());
assert_eq!(encoded.last().unwrap() & 0x80, 0);
}
}
#[test]
fn test_sleb128_roundtrip_small() {
for value in -500i64..500 {
let encoded = encode_sleb128(value);
assert!(!encoded.is_empty());
}
}
#[test]
fn test_lowering_reg_gpr32_creation() {
let reg = X86LoweringReg::gpr32(0); assert_eq!(reg.reg_num, 0);
assert_eq!(reg.reg_class, X86LoweringRegClass::GPR32);
}
#[test]
fn test_lowering_reg_xmm_creation() {
let reg = X86LoweringReg::xmm(7); assert_eq!(reg.reg_num, 7);
assert_eq!(reg.reg_class, X86LoweringRegClass::XMM);
}
#[test]
fn test_reg_class_segment_size() {
assert_eq!(X86LoweringRegClass::Segment.size_bytes(), 2);
assert_eq!(X86LoweringRegClass::Flags.size_bytes(), 1);
}
#[test]
fn test_call_site_clone() {
let site = X86CallSiteInfo {
callee: "test".to_string(),
arg_stack_size: 32,
tail_call_eligible: true,
..Default::default()
};
let cloned = site.clone();
assert_eq!(cloned.callee, "test");
assert_eq!(cloned.arg_stack_size, 32);
assert!(cloned.tail_call_eligible);
}
#[test]
fn test_lowering_result_clone() {
let result = X86LoweringResult::default();
let cloned = result.clone();
assert_eq!(cloned.stats.functions_lowered, 0);
}
#[test]
fn test_abi_matrix_sysv64_int_return() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let result = lowering.lower_function(
"sysv_int",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(result.ret_lowered.as_ref().unwrap().regs[0].reg_num, 0);
assert_eq!(
result.ret_lowered.as_ref().unwrap().regs[0].reg_class,
X86LoweringRegClass::GPR64
);
}
#[test]
fn test_abi_matrix_sysv64_fp_return() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let result = lowering.lower_function("sysv_fp", &[], Some(&make_test_ret(8, true)));
assert_eq!(
result.ret_lowered.as_ref().unwrap().regs[0].reg_class,
X86LoweringRegClass::XMM
);
}
#[test]
fn test_abi_matrix_win64_int_return() {
let mut lowering = X86Lowering::new_x86_64_win64();
let result = lowering.lower_function(
"win_int",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(result.ret_lowered.as_ref().unwrap().regs[0].reg_num, 0);
}
#[test]
fn test_abi_matrix_win64_fp_return() {
let mut lowering = X86Lowering::new_x86_64_win64();
let result = lowering.lower_function("win_fp", &[], Some(&make_test_ret(8, true)));
assert_eq!(
result.ret_lowered.as_ref().unwrap().regs[0].reg_class,
X86LoweringRegClass::XMM
);
}
#[test]
fn test_abi_matrix_cdecl_int_return() {
let mut lowering = X86Lowering::new_x86_32_cdecl();
let result = lowering.lower_function(
"cdecl_int",
&[make_test_arg(4)],
Some(&make_test_ret(4, false)),
);
assert_eq!(
result.ret_lowered.as_ref().unwrap().regs[0].reg_class,
X86LoweringRegClass::GPR32
);
}
#[test]
fn test_abi_matrix_cdecl_fp_return() {
let mut lowering = X86Lowering::new_x86_32_cdecl();
let result = lowering.lower_function("cdecl_fp", &[], Some(&make_test_ret(4, true)));
assert_eq!(
result.ret_lowered.as_ref().unwrap().regs[0].reg_class,
X86LoweringRegClass::XMM
);
}
#[test]
fn test_abi_matrix_fastcall_two_reg_args() {
let mut lowering = X86Lowering::new(X86TargetVariant::X86_32)
.with_calling_convention(X86CallingConvention::Fast);
let result = lowering.lower_function(
"fast",
&[make_test_arg(4), make_test_arg(4)],
Some(&make_test_ret(4, false)),
);
assert_eq!(result.args_lowered.len(), 2);
}
#[test]
fn test_abi_matrix_vectorcall_x64_vector_first() {
let mut lowering = X86Lowering::new(X86TargetVariant::X86_64)
.with_calling_convention(X86CallingConvention::VectorCall);
let vec_arg = X86LoweringArg {
size: 16,
alignment: 16,
..Default::default()
};
let result = lowering.lower_function("vec", &[vec_arg], None);
assert_eq!(result.args_lowered.len(), 1);
}
#[test]
fn test_abi_matrix_regcall_many_gprs() {
let mut lowering = X86Lowering::new(X86TargetVariant::X86_64)
.with_calling_convention(X86CallingConvention::X86_RegCall);
let args: Vec<X86LoweringArg> = (0..11).map(|_| make_test_arg(8)).collect();
let result = lowering.lower_function("regcall", &args, None);
assert_eq!(result.args_lowered.len(), 11);
}
#[test]
fn test_x32_abi_pointer_width() {
let x32 = X86TargetVariant::X32;
assert_eq!(x32.pointer_width(), 32);
assert!(x32.is_64bit_mode());
}
#[test]
fn test_x32_lowering_uses_64bit_regs() {
let lowering = X86Lowering::new(X86TargetVariant::X32);
assert_eq!(lowering.target_variant.register_width(), 64);
assert_eq!(lowering.frame_lowering.push_size, 8);
}
#[test]
fn test_x16_minimal_support() {
let lowering = X86Lowering::new(X86TargetVariant::X86_16);
assert_eq!(lowering.target_variant.pointer_width(), 16);
assert!(!lowering.has_sse2); }
#[test]
fn test_sysv64_struct_two_eightbytes_integer() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let arg = X86LoweringArg {
size: 16,
alignment: 8,
..Default::default()
};
let result = al.lower_single_arg(&arg);
assert_eq!(result.regs.len(), 2);
assert_eq!(result.regs[0].reg_num, 7); assert_eq!(result.regs[1].reg_num, 6); }
#[test]
fn test_sysv64_struct_three_eightbytes_stack() {
let mut al =
X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let arg = X86LoweringArg {
size: 24,
alignment: 8,
..Default::default()
};
let result = al.lower_single_arg(&arg);
assert!(result.stack_offset.is_some());
}
#[test]
fn test_win64_struct_always_indirect() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let arg = X86LoweringArg {
size: 16,
alignment: 8,
is_byval: true,
..Default::default()
};
let result = al.lower_single_arg(&arg);
assert!(!result.regs.is_empty()); }
#[test]
fn test_sysv64_classification_integer_class() {
let arg = X86LoweringArg {
size: 8,
alignment: 8,
..Default::default()
};
assert!(arg.size <= 8);
}
#[test]
fn test_sysv64_classification_sse_class() {
let fp_arg = X86LoweringArg {
size: 8,
alignment: 8,
type_id: Some(65),
..Default::default()
};
assert_eq!(fp_arg.size, 8); }
#[test]
fn test_sysv64_classification_memory_class() {
let large = X86LoweringArg {
size: 32,
alignment: 8,
..Default::default()
};
assert!(large.size > 16); }
#[test]
fn test_prologue_preserves_stack_alignment() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let frame = fl.compute_frame_layout();
assert_eq!(frame.frame_size % 16, 0);
}
#[test]
fn test_prologue_sub_instruction_present() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_prologue(128);
let has_sub = instrs.iter().any(|i| i.opcode == MOpcode::G_SUB);
assert!(has_sub);
}
#[test]
fn test_epilogue_ret_is_terminator() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_epilogue(128);
let last = instrs.last().unwrap();
assert!(last.flags.is_terminator);
assert!(last.flags.is_return);
assert!(last.flags.is_barrier);
}
#[test]
fn test_prologue_epilogue_pairing() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
for frame_size in [0, 16, 32, 64, 128, 256, 1024].iter() {
let pro = fl.emit_prologue(*frame_size);
let epi = fl.emit_epilogue(*frame_size);
assert!(!pro.is_empty());
assert!(!epi.is_empty());
}
}
#[test]
fn test_red_zone_leaf_small_frame_eligible() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(fl.can_use_red_zone(true, 64));
assert!(fl.can_use_red_zone(true, 128));
}
#[test]
fn test_red_zone_leaf_large_frame_not_eligible() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(!fl.can_use_red_zone(true, 256));
assert!(!fl.can_use_red_zone(true, 1024));
}
#[test]
fn test_red_zone_non_leaf_never_eligible() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
assert!(!fl.can_use_red_zone(false, 0));
assert!(!fl.can_use_red_zone(false, 16));
assert!(!fl.can_use_red_zone(false, 64));
}
#[test]
fn test_cet_endbr_at_function_entry() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64)
.with_shadow_stack(true)
.with_endbr(true);
ss.lower();
assert_eq!(ss.prologue[0].opcode, MOpcode::G_INTRINSIC);
}
#[test]
fn test_cet_incsspq_after_endbr() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64)
.with_shadow_stack(true)
.with_endbr(true);
ss.lower();
assert_eq!(ss.prologue[1].opcode, MOpcode::G_INTRINSIC);
assert!(ss.prologue[1].flags.has_side_effects);
}
#[test]
fn test_cet_save_return_address() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64).with_shadow_stack(true);
ss.lower();
let has_save = ss
.prologue
.iter()
.any(|i| i.opcode == MOpcode::G_STORE && i.flags.has_side_effects);
assert!(has_save);
}
#[test]
fn test_cet_epilogue_restores_shadow_stack() {
let mut ss = X86ShadowStack::new(X86TargetVariant::X86_64).with_shadow_stack(true);
ss.lower();
assert!(!ss.epilogue.is_empty());
assert_eq!(ss.epilogue[0].opcode, MOpcode::G_INTRINSIC);
}
#[test]
fn test_eh_catch_all_dispatch() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 0,
catch_types: vec![0], cleanup: false,
filter_types: vec![],
}];
ehl.lower_landing_pads(&pads);
let lowered = &ehl.landing_pads[0];
assert!(lowered.catches[0].is_catch_all);
}
#[test]
fn test_eh_catch_and_cleanup_combined() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 0,
catch_types: vec![1, 2],
cleanup: true,
filter_types: vec![],
}];
ehl.lower_landing_pads(&pads);
let lowered = &ehl.landing_pads[0];
assert_eq!(lowered.catches.len(), 2);
assert!(lowered.has_cleanup);
assert!(lowered.cleanup_block.is_some());
}
#[test]
fn test_eh_multiple_filter_dispatch() {
let mut ehl = X86EHPadLowering::new(X86TargetVariant::X86_64);
let pads = vec![LandingPadDescriptor {
block_id: 0,
catch_types: vec![],
cleanup: false,
filter_types: vec![1, 2, 3],
}];
ehl.lower_landing_pads(&pads);
let lowered = &ehl.landing_pads[0];
assert_eq!(lowered.filters.len(), 3);
}
#[test]
fn test_cfi_cie_has_correct_structure() {
let cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
assert!(cfi.cie.is_cie);
assert_eq!(cfi.cie.version, 1);
assert!(!cfi.cie.augmentation.is_empty());
}
#[test]
fn test_cfi_fde_inherits_from_cie() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.begin_fde(0x1000, 256);
let fde = &cfi.fdes[0];
assert!(!fde.is_cie);
assert_eq!(fde.version, cfi.cie.version);
assert_eq!(fde.augmentation, cfi.cie.augmentation);
assert_eq!(fde.return_address_register, cfi.cie.return_address_register);
}
#[test]
fn test_cfi_standard_prologue_cfa_sequence() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
cfi.emit_cfi();
assert_eq!(cfi.current_cfa_register, DWARF_RSP);
assert_eq!(cfi.current_cfa_offset, 8);
cfi.emit_def_cfa(DWARF_RSP, 16);
cfi.emit_offset(DWARF_RBP, -16);
cfi.emit_def_cfa_register(DWARF_RBP);
assert_eq!(cfi.current_cfa_register, DWARF_RBP);
assert_eq!(cfi.current_cfa_offset, 16);
}
#[test]
fn test_cfi_register_save_for_all_callee_saved() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
let saved_regs = [DWARF_RBX, DWARF_R12, DWARF_R13, DWARF_R14, DWARF_R15];
for (i, ®) in saved_regs.iter().enumerate() {
cfi.emit_offset(reg, -((i + 2) as i32 * 8)); }
assert_eq!(cfi.register_save_locations.len(), saved_regs.len());
}
#[test]
fn test_uleb128_all_one_byte_values() {
for value in 0u64..128 {
let encoded = encode_uleb128(value);
assert_eq!(encoded.len(), 1);
assert_eq!(encoded[0] as u64, value);
}
}
#[test]
fn test_uleb128_two_byte_boundary() {
assert_eq!(encode_uleb128(127).len(), 1);
assert_eq!(encode_uleb128(128).len(), 2);
assert_eq!(encode_uleb128(16383).len(), 2);
assert_eq!(encode_uleb128(16384).len(), 3);
}
#[test]
fn test_sleb128_one_byte_values() {
for value in -64i64..64 {
let encoded = encode_sleb128(value);
assert_eq!(encoded.len(), 1, "Failed for value: {}", value);
}
}
#[test]
fn test_sleb128_two_byte_transition() {
assert_eq!(encode_sleb128(63).len(), 1);
assert_eq!(encode_sleb128(64).len(), 2);
assert_eq!(encode_sleb128(-64).len(), 1);
assert_eq!(encode_sleb128(-65).len(), 2);
}
#[test]
fn test_terminator_instructions_have_correct_flags() {
let ret_instr = X86MachineInstr {
opcode: MOpcode::G_RETURN,
defs: vec![],
uses: vec![],
flags: X86MIFlags {
is_terminator: true,
is_return: true,
is_barrier: true,
..Default::default()
},
};
assert!(ret_instr.flags.is_terminator);
assert!(ret_instr.flags.is_return);
assert!(ret_instr.flags.is_barrier);
}
#[test]
fn test_call_instruction_has_side_effects() {
let call_instr = X86MachineInstr {
opcode: MOpcode::G_CALL,
defs: vec![],
uses: vec![],
flags: X86MIFlags {
is_call: true,
has_side_effects: true,
..Default::default()
},
};
assert!(call_instr.flags.is_call);
assert!(call_instr.flags.has_side_effects);
}
#[test]
fn test_load_store_instructions_have_memory_flags() {
let load = X86MachineInstr {
opcode: MOpcode::G_LOAD,
defs: vec![VReg(1)],
uses: vec![],
flags: X86MIFlags {
may_load: true,
..Default::default()
},
};
let store = X86MachineInstr {
opcode: MOpcode::G_STORE,
defs: vec![],
uses: vec![X86MachineOperand::vreg(VReg(2))],
flags: X86MIFlags {
may_store: true,
..Default::default()
},
};
assert!(load.flags.may_load);
assert!(!load.flags.may_store);
assert!(store.flags.may_store);
assert!(!store.flags.may_load);
}
#[test]
fn test_branch_instruction_flags() {
let br_cond = X86MachineInstr {
opcode: MOpcode::G_BRCOND,
defs: vec![],
uses: vec![
X86MachineOperand::vreg(VReg(1)),
X86MachineOperand::block(5),
],
flags: X86MIFlags {
is_branch: true,
..Default::default()
},
};
assert!(br_cond.flags.is_branch);
}
#[test]
fn test_frame_layout_offsets_are_consistent() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let layout = fl.compute_frame_layout();
assert_eq!(layout.return_address_offset, 0);
assert_eq!(layout.saved_rbp_offset, 8);
assert_eq!(layout.callee_saved_start, 16);
}
#[test]
fn test_frame_layout_32bit_offsets() {
let fl = X86FrameLoweringEngine::new(X86TargetVariant::X86_32, X86CallingConvention::C);
let layout = fl.compute_frame_layout();
assert_eq!(layout.return_address_offset, 0);
assert_eq!(layout.saved_rbp_offset, 4);
assert_eq!(layout.callee_saved_start, 8);
}
#[test]
fn test_frame_layout_without_frame_pointer() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
fl.use_frame_pointer = false;
let layout = fl.compute_frame_layout();
assert_eq!(layout.callee_saved_start, 8);
}
#[test]
fn test_frame_layout_alignment() {
let fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let layout = fl.compute_frame_layout();
assert_eq!(layout.frame_size % 16, 0);
}
#[test]
fn test_stress_many_small_functions() {
let mut lowering = X86Lowering::new_x86_64_sysv();
for i in 0..100 {
let name = format!("func_{}", i);
lowering.begin_function(&name);
let _ = lowering.lower_function(&name, &[make_test_arg(8)], None);
}
assert_eq!(lowering.stats.functions_lowered, 100);
assert_eq!(lowering.stats.args_lowered, 100);
}
#[test]
fn test_stress_deep_arg_list() {
let mut al = X86ArgLowering::default();
let args: Vec<X86LoweringArg> = (0..100)
.map(|i| X86LoweringArg {
size: 8,
alignment: 8,
stack_offset: if i >= 6 {
Some((i - 6) as u32 * 8)
} else {
None
},
..Default::default()
})
.collect();
let lowered = al.lower_arguments(&args);
assert_eq!(lowered.len(), 100);
}
#[test]
fn test_stress_all_calling_conventions_pipeline() {
let all_ccs = vec![
X86CallingConvention::X86_64_SysV,
X86CallingConvention::Win64,
X86CallingConvention::C,
X86CallingConvention::Fast,
X86CallingConvention::StdCall,
X86CallingConvention::ThisCall,
X86CallingConvention::VectorCall,
X86CallingConvention::X86_RegCall,
];
for cc in all_ccs {
let variant = if cc.is_64bit() {
X86TargetVariant::X86_64
} else {
X86TargetVariant::X86_32
};
let mut lowering = X86Lowering::new(variant).with_calling_convention(cc);
let result = lowering.lower_function(
"test",
&[make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(result.stats.functions_lowered, 1);
}
}
#[test]
fn test_stress_all_opt_levels_pipeline() {
let opt_levels = vec![
X86OptLevel::O0,
X86OptLevel::O1,
X86OptLevel::O2,
X86OptLevel::O3,
X86OptLevel::Os,
X86OptLevel::Oz,
];
for opt in opt_levels {
let mut lowering = X86Lowering::new_x86_64_sysv().with_opt_level(opt);
let result = lowering.lower_function("test", &[make_test_arg(8)], None);
assert_eq!(result.stats.functions_lowered, 1);
}
}
#[test]
fn test_stress_all_lowering_modes() {
let modes = vec![
X86LoweringMode::SelectionDAG,
X86LoweringMode::FastISel,
X86LoweringMode::GlobalISel,
];
for mode in modes {
let mut lowering = X86Lowering::new_x86_64_sysv().with_mode(mode);
let result = lowering.lower_function("test", &[make_test_arg(8)], None);
assert_eq!(result.stats.functions_lowered, 1);
}
}
#[test]
fn test_regression_double_lower_same_function() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let r1 = lowering.lower_function("f", &[make_test_arg(8)], None);
let r2 = lowering.lower_function("f", &[make_test_arg(8)], None);
assert_eq!(r1.args_lowered.len(), r2.args_lowered.len());
}
#[test]
fn test_regression_reset_arg_lowering_mid_pipeline() {
let mut al = X86ArgLowering::default();
let _ = al.lower_arguments(&[make_test_arg(8), make_test_arg(8)]);
assert_eq!(al.int_regs_used, 2);
al.reset();
assert_eq!(al.int_regs_used, 0);
let _ = al.lower_arguments(&[make_test_arg(8)]);
assert_eq!(al.int_regs_used, 1);
}
#[test]
fn test_regression_cfi_after_multiple_fdes() {
let mut cfi = X86DwarfCFI::new(X86TargetVariant::X86_64);
for i in 0..10 {
cfi.begin_fde(i * 0x1000, 256);
cfi.emit_def_cfa(DWARF_RBP, 16);
cfi.end_fde();
}
assert_eq!(cfi.fdes.len(), 10);
}
#[test]
fn test_regression_call_lowering_after_frame_lowering() {
let mut cl = X86CallLowering::default();
let site = cl.lower_sysv64_call("target", &[make_test_arg(8)], None);
assert!(!site.callee.is_empty());
}
#[test]
fn test_ir_transformer_performs_batched_expansion() {
let mut transformer = X86IRTransformer::default();
let result = transformer.transform();
assert!(result.instructions_transformed > 0);
assert!(result.intrinsics_expanded > 0);
assert!(result.target_intrinsics_expanded > 0);
}
#[test]
fn test_frame_lowering_no_unnecessary_instructions() {
let mut fl = X86FrameLoweringEngine::new(
X86TargetVariant::X86_64,
X86CallingConvention::X86_64_SysV,
);
let instrs = fl.emit_prologue(0);
let sub_count = instrs.iter().filter(|i| i.opcode == MOpcode::G_SUB).count();
assert_eq!(sub_count, 0);
}
#[test]
fn test_lowering_reuses_components() {
let lowering = X86Lowering::default();
assert!(!lowering.ir_transformer.supported_expansions.is_empty());
assert!(lowering.callee_saved.callee_saved_regs.len() > 0);
assert!(lowering.dwarf_cfi.is_enabled());
}
#[test]
fn test_win64_no_red_zone_by_default() {
let lowering = X86Lowering::new_x86_64_win64();
assert!(!lowering.use_red_zone);
assert!(!lowering.frame_lowering.has_red_zone);
}
#[test]
fn test_sysv64_uses_red_zone() {
let lowering = X86Lowering::new_x86_64_sysv();
assert!(lowering.use_red_zone);
assert!(lowering.frame_lowering.has_red_zone);
}
#[test]
fn test_win64_has_shadow_space() {
let cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
assert_eq!(cl.shadow_store_size, WIN64_SHADOW_SPACE);
}
#[test]
fn test_sysv64_no_shadow_space() {
let cl = X86CallLowering::new(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
assert_eq!(cl.shadow_store_size, 0);
}
#[test]
fn test_smoke_create_all_factory_functions() {
let _ = create_x86_64_sysv_lowering();
let _ = create_x86_64_win64_lowering();
let _ = create_x86_32_cdecl_lowering();
let _ = create_x86_ir_transformer(X86TargetVariant::X86_64);
let _ =
create_x86_call_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let _ =
create_x86_arg_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let _ =
create_x86_ret_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let _ =
create_x86_frame_lowering(X86TargetVariant::X86_64, X86CallingConvention::X86_64_SysV);
let _ = create_x86_callee_saved(X86TargetVariant::X86_64);
let _ = create_x86_shadow_stack(X86TargetVariant::X86_64);
let _ = create_x86_eh_pad_lowering(X86TargetVariant::X86_64);
let _ = create_x86_dwarf_cfi(X86TargetVariant::X86_64);
}
#[test]
fn test_smoke_full_lowering_roundtrip() {
let mut lowering = X86Lowering::new_x86_64_sysv();
let transform = lowering.ir_transformer.transform();
assert!(transform.instructions_transformed > 0);
let calls = lowering.lower_calls_only();
assert_eq!(calls, 0);
let frame = lowering.lower_frame_only();
assert!(frame.alignment > 0);
let regs = lowering.get_callee_saved_regs();
assert!(regs.is_empty()); }
#[test]
fn test_smoke_lower_function_with_all_features() {
let mut lowering = X86Lowering::new_x86_64_sysv()
.with_avx(true)
.with_cet(true)
.with_frame_pointer_elim(false)
.with_red_zone(true);
lowering.begin_function("all_features");
if let Some(ref mut state) = lowering.function_state {
state.uses_eh = true;
state.landing_pads.push(LandingPadDescriptor {
block_id: 0,
catch_types: vec![1],
cleanup: true,
filter_types: vec![],
});
}
let result = lowering.lower_function(
"all_features",
&[make_test_arg(8), make_test_arg(8), make_test_arg(8)],
Some(&make_test_ret(8, false)),
);
assert_eq!(result.stats.functions_lowered, 1);
assert_eq!(result.stats.args_lowered, 3);
assert_eq!(result.stats.returns_lowered, 1);
assert_eq!(result.stats.landing_pads_lowered, 1);
assert_eq!(result.stats.shadow_stack_instructions, 2);
assert!(result.stats.cfi_directives_emitted > 0);
}
#[test]
fn test_max_u32_align_to() {
assert_eq!(align_to(u32::MAX - 16, 16), u32::MAX - 15);
}
#[test]
fn test_align_to_zero_alignment() {
assert_eq!(align_to(42, 0), 42);
assert_eq!(align_to(0, 0), 0);
}
#[test]
fn test_fits_i8_boundaries() {
assert!(fits_i8(-128));
assert!(fits_i8(127));
assert!(!fits_i8(-129));
assert!(!fits_i8(128));
}
#[test]
fn test_fits_i32_boundaries() {
assert!(fits_i32(-2147483648));
assert!(fits_i32(2147483647));
assert!(!fits_i32(-2147483649));
assert!(!fits_i32(2147483648));
}
#[test]
fn test_type_size_bytes_all_known_types() {
assert_eq!(type_size_bytes(0), 0);
assert_eq!(type_size_bytes(1), 1);
assert_eq!(type_size_bytes(8), 1);
assert_eq!(type_size_bytes(16), 2);
assert_eq!(type_size_bytes(32), 4);
assert_eq!(type_size_bytes(33), 4); assert_eq!(type_size_bytes(64), 8);
assert_eq!(type_size_bytes(65), 8); assert_eq!(type_size_bytes(80), 10); assert_eq!(type_size_bytes(128), 16); }
#[test]
fn test_uleb128_max_u32() {
let encoded = encode_uleb128(u32::MAX as u64);
assert!(!encoded.is_empty());
assert_eq!(encoded.len(), 5); }
#[test]
fn test_sleb128_min_i32() {
let encoded = encode_sleb128(i32::MIN as i64);
assert!(!encoded.is_empty());
assert_eq!(encoded.len(), 5);
}
#[test]
fn test_sleb128_max_i32() {
let encoded = encode_sleb128(i32::MAX as i64);
assert!(!encoded.is_empty());
assert_eq!(encoded.len(), 5);
}
#[test]
fn test_reg_assignment_sysv64_first_int_is_rdi() {
let mut al = X86ArgLowering::default();
let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 7); }
#[test]
fn test_reg_assignment_sysv64_second_int_is_rsi() {
let mut al = X86ArgLowering::default();
let _ = al.lower_single_arg(&make_test_arg(8)); let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 6); }
#[test]
fn test_reg_assignment_sysv64_third_is_rdx() {
let mut al = X86ArgLowering::default();
let _ = al.lower_single_arg(&make_test_arg(8)); let _ = al.lower_single_arg(&make_test_arg(8)); let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 2); }
#[test]
fn test_reg_assignment_sysv64_fourth_is_rcx() {
let mut al = X86ArgLowering::default();
let _ = al.lower_single_arg(&make_test_arg(8)); let _ = al.lower_single_arg(&make_test_arg(8)); let _ = al.lower_single_arg(&make_test_arg(8)); let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 1); }
#[test]
fn test_reg_assignment_sysv64_fifth_is_r8() {
let mut al = X86ArgLowering::default();
for _ in 0..4 {
let _ = al.lower_single_arg(&make_test_arg(8));
}
let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 8); }
#[test]
fn test_reg_assignment_sysv64_sixth_is_r9() {
let mut al = X86ArgLowering::default();
for _ in 0..5 {
let _ = al.lower_single_arg(&make_test_arg(8));
}
let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 9); }
#[test]
fn test_reg_assignment_sysv64_seventh_is_stack() {
let mut al = X86ArgLowering::default();
for _ in 0..6 {
let _ = al.lower_single_arg(&make_test_arg(8));
}
let result = al.lower_single_arg(&make_test_arg(8));
assert!(result.stack_offset.is_some());
}
#[test]
fn test_reg_assignment_win64_first_is_rcx() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 1); }
#[test]
fn test_reg_assignment_win64_second_is_rdx() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
let _ = al.lower_single_arg(&make_test_arg(8)); let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 2); }
#[test]
fn test_reg_assignment_win64_third_is_r8() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
for _ in 0..2 {
let _ = al.lower_single_arg(&make_test_arg(8));
}
let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 8); }
#[test]
fn test_reg_assignment_win64_fourth_is_r9() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
for _ in 0..3 {
let _ = al.lower_single_arg(&make_test_arg(8));
}
let result = al.lower_single_arg(&make_test_arg(8));
assert_eq!(result.regs[0].reg_num, 9); }
#[test]
fn test_reg_assignment_win64_fifth_is_stack() {
let mut al = X86ArgLowering::new(X86TargetVariant::X86_64, X86CallingConvention::Win64);
for _ in 0..4 {
let _ = al.lower_single_arg(&make_test_arg(8));
}
let result = al.lower_single_arg(&make_test_arg(8));
assert!(result.stack_offset.is_some());
}
#[test]
fn test_x86_lowering_reg_display() {
let reg = X86LoweringReg::gpr64(0);
assert_eq!(reg.reg_num, 0);
assert_eq!(reg.reg_class, X86LoweringRegClass::GPR64);
}
#[test]
fn test_frame_layout_debug_fields() {
let layout = X86FrameLayout::default();
let _ = layout.frame_size;
let _ = layout.alignment;
let _ = layout.has_red_zone;
let _ = layout.red_zone_size;
let _ = layout.has_frame_pointer;
let _ = layout.needs_realignment;
let _ = layout.has_var_sized_objects;
let _ = layout.is_seh_frame;
}
}