use std::collections::HashMap;
use std::sync::Arc;
use crate::backend::Reg;
use crate::compiler::compiled_code::{CallSite, CompiledCode, Symbol};
use crate::compiler::graph::cfg::liveness::LiveIntervalValueCategory;
use crate::compiler::graph::cfg::CFG;
use crate::compiler::graph::{Const, Literal, NodeId, Type};
use crate::compiler::Compiler;
use dynasmrt::x64::X64Relocation;
use dynasmrt::VecAssembler;
use dynasmrt::{dynasm, DynamicLabel, DynasmApi, DynasmLabelApi};
use super::call_sites::X64StaticCallSite;
use super::compiled_code::X64CompiledCode;
use super::{X64Reg, X64};
type Op = crate::compiler::graph::BaseOp;
const TMP0: u8 = X64Reg::R14 as usize as u8;
const TMP1: u8 = X64Reg::R15 as usize as u8;
pub struct CodeGen<'c, 'cfg> {
compiler: &'c Compiler,
symbol: Symbol,
cfg: &'cfg mut CFG<Op>,
ops: VecAssembler<X64Relocation>,
labels: Vec<DynamicLabel>,
current_block_id: usize,
static_call_sites: HashMap<usize, Box<X64StaticCallSite>>,
rsp_delta: i32,
extra_push: bool,
constants: HashMap<Const, DynamicLabel>,
}
impl<'c, 'cfg> CodeGen<'c, 'cfg> {
pub fn new(compiler: &'c Compiler, symbol: Symbol, cfg: &'cfg mut CFG<Op>) -> Self {
Self {
compiler,
symbol,
cfg,
ops: VecAssembler::new(0),
labels: vec![],
current_block_id: 0,
static_call_sites: Default::default(),
rsp_delta: 0,
extra_push: false,
constants: Default::default(),
}
}
fn register_constant(&mut self, constant: Const) -> DynamicLabel {
if self.constants.contains_key(&constant) {
self.constants[&constant]
} else {
let label = self.ops.new_dynamic_label();
self.constants.insert(constant, label);
label
}
}
fn record_static_callsite(&mut self, symbol: Symbol, return_address_offset: usize) {
self.static_call_sites.insert(
return_address_offset,
Box::new(X64StaticCallSite::new(symbol, return_address_offset)),
);
}
fn gen_prologue(&mut self) {
dynasm! {self.ops
; push rbp
; mov rbp, rsp
}
let mut rsp_offset = 8usize;
let mut num_callee_saved_regs = 0;
for r in self.cfg.used_registers.iter_ones() {
let r: X64Reg = r.into();
if r.is_callee_saved() {
assert!(!r.is_fpr());
num_callee_saved_regs += 1;
dynasm!(self.ops; push Rq(r as u8));
rsp_offset += 8;
}
}
self.rsp_delta = self.cfg.stack_size as i32;
if (num_callee_saved_regs & 1) != 0 {
self.rsp_delta += 8;
}
if self.rsp_delta != 0 {
dynasm!(self.ops; sub rsp, self.rsp_delta);
rsp_offset += self.rsp_delta as usize;
}
assert_eq!((rsp_offset & 0b111), 0);
if (rsp_offset & 0b1111) == 0 {
dynasm!(self.ops; push rax);
self.extra_push = true;
}
}
fn gen_epilogue(&mut self) {
if self.extra_push {
dynasm!(self.ops; pop rcx);
}
if self.rsp_delta != 0 {
dynasm!(self.ops; add rsp, self.rsp_delta);
}
for r in self.cfg.used_registers.iter_ones().rev() {
let r: X64Reg = r.into();
if r.is_callee_saved() {
assert!(!r.is_fpr());
dynasm!(self.ops; pop Rq(r as u8));
}
}
dynasm! {self.ops
; mov rsp, rbp
; pop rbp
; ret
}
}
fn gen_mov_reg_reg(&mut self, t: Type, dst: u8, src: u8) {
if dst == src {
return;
}
match t {
Type::Bool | Type::I8 => dynasm!(self.ops; mov Rb(dst), Rb(src)),
Type::I16 => dynasm!(self.ops; mov Rw(dst), Rw(src)),
Type::I32 => dynasm!(self.ops; mov Rd(dst), Rd(src)),
Type::I64 => dynasm!(self.ops; mov Rq(dst), Rq(src)),
Type::F32 => dynasm!(self.ops; movss Rx(dst), Rx(src)),
Type::F64 => dynasm!(self.ops; movsd Rx(dst), Rx(src)),
Type::Void | Type::Top => unreachable!(),
}
}
fn gen_push(&mut self, n: NodeId, stack_arg_index: i32, total_stack_arg_size: i32) {
let reg = if let Some(reg) = self.cfg.liveness[self.cfg.g[n].interval].reg {
reg as u8
} else {
let mem = self.cfg.liveness[self.cfg.g[n].interval].mem;
self.gen_mov_reg_stack(self.cfg.g[n].ty, TMP0, mem);
TMP0
};
let offset = -total_stack_arg_size + (stack_arg_index << 3);
match self.cfg.g[n].ty {
Type::Bool | Type::I8 | Type::I16 | Type::I32 | Type::I64 => {
dynasm!(self.ops; mov [rsp + offset], Rq(reg))
}
Type::F32 => {
dynasm!(self.ops; movss [rsp + offset], Rx(reg));
}
Type::F64 => {
dynasm!(self.ops; movsd [rsp + offset], Rx(reg));
}
Type::Void | Type::Top => unreachable!(),
}
}
fn gen_mov_reg_stack(&mut self, t: Type, r: u8, offset: i32) {
match t {
Type::Bool | Type::I8 => dynasm!(self.ops; mov Rb(r), [rsp + offset]),
Type::I16 => dynasm!(self.ops; mov Rw(r), [rsp + offset]),
Type::I32 => dynasm!(self.ops; mov Rd(r), [rsp + offset]),
Type::I64 => dynasm!(self.ops; mov Rq(r), [rsp + offset]),
Type::F32 => dynasm!(self.ops; movss Rx(r), [rsp + offset]),
Type::F64 => dynasm!(self.ops; movsd Rx(r), [rsp + offset]),
Type::Void | Type::Top => unreachable!(),
}
}
fn gen_mov_stack_reg(&mut self, t: Type, r: u8, offset: i32) {
match t {
Type::Bool | Type::I8 => dynasm!(self.ops; mov [rsp + offset], Rb(r)),
Type::I16 => dynasm!(self.ops; mov [rsp + offset], Rw(r)),
Type::I32 => dynasm!(self.ops; mov [rsp + offset], Rd(r)),
Type::I64 => dynasm!(self.ops; mov [rsp + offset], Rq(r)),
Type::F32 => dynasm!(self.ops; movss [rsp + offset], Rx(r)),
Type::F64 => dynasm!(self.ops; movsd [rsp + offset], Rx(r)),
Type::Void | Type::Top => unreachable!(),
}
}
fn save_interval(&mut self, i: usize) {
let i = &self.cfg.liveness[i];
use LiveIntervalValueCategory::*;
let t = match (i.value_category.unwrap(), i.max_mem_size) {
(Int, 1) => Type::I8,
(Int, 2) => Type::I16,
(Int, 4) => Type::I32,
(Int, 8) => Type::I64,
(Float, 4) => Type::F32,
(Float, 8) => Type::F64,
x => unreachable!("{:?}", x),
};
self.gen_mov_stack_reg(t, i.reg.unwrap() as u8, i.mem);
}
fn restore_interval(&mut self, i: usize) {
let i = &self.cfg.liveness[i];
use LiveIntervalValueCategory::*;
let t = match (i.value_category.unwrap(), i.max_mem_size) {
(Int, 1) => Type::I8,
(Int, 2) => Type::I16,
(Int, 4) => Type::I32,
(Int, 8) => Type::I64,
(Float, 4) => Type::F32,
(Float, 8) => Type::F64,
x => unreachable!("{:?}", x),
};
self.gen_mov_reg_stack(t, i.reg.unwrap() as u8, i.mem);
}
fn save_reg_if_active(&mut self, node: NodeId, reg: usize) {
for i in 0..self.cfg.liveness.len() {
let ival = &self.cfg.liveness[i];
if i != self.cfg.g[node].interval
&& ival.covers(self.cfg.g[node].cfg_id)
&& ival.reg == Some(reg)
{
self.save_interval(i);
}
}
}
fn restore_reg_if_active(&mut self, node: NodeId, reg: usize) {
for i in 0..self.cfg.liveness.len() {
let ival = &self.cfg.liveness[i];
if i != self.cfg.g[node].interval
&& ival.covers(self.cfg.g[node].cfg_id)
&& ival.reg == Some(reg)
{
self.restore_interval(i);
}
}
}
fn load_input_reg(&mut self, node: NodeId, tmp: u8) -> u8 {
let interval_id = self.cfg.g[node].interval;
assert_ne!(interval_id, usize::MAX);
if let Some(reg) = self.cfg.liveness[interval_id].reg {
reg as _
} else if let Some(fixed_reg) = self.cfg.g[node].fixed_reg {
self.save_reg_if_active(node, fixed_reg);
let mem = self.cfg.liveness[interval_id].mem;
self.gen_mov_reg_stack(self.cfg.g[node].ty, fixed_reg as u8, mem);
fixed_reg as _
} else {
let mem = self.cfg.liveness[interval_id].mem;
self.cfg.g[node].temp_reg = Some(tmp);
self.gen_mov_reg_stack(self.cfg.g[node].ty, tmp, mem);
tmp
}
}
fn load_output_reg(&mut self, node: NodeId, tmp: u8) -> u8 {
let interval_id = self.cfg.g[node].interval;
assert_ne!(
interval_id,
usize::MAX,
"Node does not have an reg assiged: {:?}",
node
);
if let Some(reg) = self.cfg.liveness[interval_id].reg {
reg as _
} else if let Some(fixed_reg) = self.cfg.g[node].fixed_reg {
self.save_reg_if_active(node, fixed_reg);
fixed_reg as _
} else {
self.cfg.g[node].temp_reg = Some(tmp);
tmp
}
}
fn update_spilled_values(&mut self, node: NodeId) {
let interval = self.cfg.g[node].interval;
let fixed_reg = self.cfg.g[node].fixed_reg;
if interval == usize::MAX {
return;
}
if let Some(_reg) = self.cfg.liveness[interval].reg {
} else if let Some(fixed_reg) = fixed_reg {
let mem = self.cfg.liveness[interval].mem;
self.gen_mov_stack_reg(self.cfg.g[node].ty, fixed_reg as u8, mem);
for index in 0..self.cfg.liveness.len() {
let ival = &self.cfg.liveness[index];
if index != interval
&& ival.covers(self.cfg.g[node].cfg_id)
&& ival.reg == Some(fixed_reg)
{
self.restore_interval(index);
}
}
} else if let Some(tmp) = self.cfg.g[node].temp_reg.take() {
let mem = self.cfg.liveness[interval].mem;
self.gen_mov_stack_reg(self.cfg.g[node].ty, tmp, mem);
}
self.cfg.g[node].temp_reg = None;
}
fn get_successor_id(&self, succ_index: usize) -> usize {
self.cfg.blocks[self.current_block_id].succs[succ_index]
}
fn get_successor_label(&self, succ_index: usize) -> DynamicLabel {
self.labels[self.get_successor_id(succ_index)]
}
fn gen_shift_op(&mut self, op: Op, node: NodeId, shift_op: X64Reg) {
let t = self.cfg.g[node].ty;
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let y = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
let mut tgt = self.load_output_reg(node, TMP1);
let original_tgt = tgt;
debug_assert_ne!(TMP1, shift_op as _);
if tgt == shift_op as _ {
tgt = TMP1;
}
if tgt != x {
self.gen_mov_reg_reg(self.cfg.g[node].ty, tgt, x);
}
debug_assert_ne!(tgt, shift_op as _, "{:?} {:?}", op, self.cfg.g[node]);
debug_assert_eq!(y, shift_op as _);
debug_assert_ne!(tgt, shift_op as _);
match (op, t) {
(Op::Shl, Type::I32) => dynasm!(self.ops; sal Rd(tgt), cl),
(Op::ShrS, Type::I32) => dynasm!(self.ops; sar Rd(tgt), cl),
(Op::ShrU, Type::I32) => dynasm!(self.ops; shr Rd(tgt), cl),
(Op::Rotl, Type::I32) => dynasm!(self.ops; rol Rd(tgt), cl),
(Op::Rotr, Type::I32) => dynasm!(self.ops; ror Rd(tgt), cl),
(Op::Shl, Type::I64) => dynasm!(self.ops; sal Rq(tgt), cl),
(Op::ShrS, Type::I64) => dynasm!(self.ops; sar Rq(tgt), cl),
(Op::ShrU, Type::I64) => dynasm!(self.ops; shr Rq(tgt), cl),
(Op::Rotl, Type::I64) => dynasm!(self.ops; rol Rq(tgt), cl),
(Op::Rotr, Type::I64) => dynasm!(self.ops; ror Rq(tgt), cl),
x => unreachable!("{:?}", x),
}
if original_tgt != tgt {
self.gen_mov_reg_reg(self.cfg.g[node].ty, original_tgt, tgt);
}
}
fn gen_binop(&mut self, op: Op, node: NodeId) {
let t = self.cfg.g[node].ty;
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let y = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
let mut tgt = self.load_output_reg(node, TMP0);
let original_tgt = tgt;
if tgt == y {
if x != TMP0 {
self.gen_mov_reg_reg(t, TMP0, x);
}
tgt = TMP0;
} else if tgt != x {
self.gen_mov_reg_reg(t, tgt, x);
}
match (op, t) {
(Op::Add, Type::I32) => dynasm!(self.ops; add Rd(tgt), Rd(y)),
(Op::Sub, Type::I32) => dynasm!(self.ops; sub Rd(tgt), Rd(y)),
(Op::Mul, Type::I32) => dynasm!(self.ops; imul Rd(tgt), Rd(y)),
(Op::And, Type::I32) => dynasm!(self.ops; and Rd(tgt), Rd(y)),
(Op::Or, Type::I32) => dynasm!(self.ops; or Rd(tgt), Rd(y)),
(Op::Xor, Type::I32) => dynasm!(self.ops; xor Rd(tgt), Rd(y)),
(Op::Add, Type::I64) => dynasm!(self.ops; add Rq(tgt), Rq(y)),
(Op::Sub, Type::I64) => dynasm!(self.ops; sub Rq(tgt), Rq(y)),
(Op::Mul, Type::I64) => dynasm!(self.ops; imul Rq(tgt), Rq(y)),
(Op::And, Type::I64) => dynasm!(self.ops; and Rq(tgt), Rq(y)),
(Op::Or, Type::I64) => dynasm!(self.ops; or Rq(tgt), Rq(y)),
(Op::Xor, Type::I64) => dynasm!(self.ops; xor Rq(tgt), Rq(y)),
(Op::Add, Type::F32) => dynasm!(self.ops; addss Rx(tgt), Rx(y)),
(Op::Sub, Type::F32) => dynasm!(self.ops; subss Rx(tgt), Rx(y)),
(Op::Mul, Type::F32) => dynasm!(self.ops; mulss Rx(tgt), Rx(y)),
(Op::FDiv, Type::F32) => dynasm!(self.ops; divss Rx(tgt), Rx(y)),
(Op::Add, Type::F64) => dynasm!(self.ops; addsd Rx(tgt), Rx(y)),
(Op::Sub, Type::F64) => dynasm!(self.ops; subsd Rx(tgt), Rx(y)),
(Op::Mul, Type::F64) => dynasm!(self.ops; mulsd Rx(tgt), Rx(y)),
(Op::FDiv, Type::F64) => dynasm!(self.ops; divsd Rx(tgt), Rx(y)),
x => unreachable!("{:?}", x),
}
if original_tgt == y {
self.gen_mov_reg_reg(t, original_tgt, TMP0);
}
}
fn gen_div(&mut self, op: Op, node: NodeId) {
let t = self.cfg.g[node].ty;
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let y = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
let tgt = self.load_output_reg(node, TMP0);
debug_assert_eq!(x, X64Reg::RAX as _);
debug_assert_ne!(y, X64Reg::RDX as _);
debug_assert_ne!(y, X64Reg::RAX as _);
debug_assert_eq!(tgt, X64Reg::RAX as _);
self.save_reg_if_active(node, X64Reg::RDX as _);
match (op, t) {
(Op::DivS, Type::I32) => {
dynasm!(self.ops; cdq);
dynasm!(self.ops; idiv Rd(y));
}
(Op::DivU, Type::I32) => {
dynasm!(self.ops; xor edx, edx);
dynasm!(self.ops; div Rd(y));
}
(Op::DivS, Type::I64) => {
dynasm!(self.ops; cqo);
dynasm!(self.ops; idiv Rq(y));
}
(Op::DivU, Type::I64) => {
dynasm!(self.ops; xor rdx, rdx);
dynasm!(self.ops; div Rq(y));
}
_ => unreachable!(),
}
self.restore_reg_if_active(node, X64Reg::RDX as _);
}
fn gen_irem(&mut self, op: Op, node: NodeId) {
let t = self.cfg.g[node].ty;
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let y = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
let tgt = self.load_output_reg(node, TMP0);
debug_assert_eq!(x, X64Reg::RAX as _);
debug_assert_ne!(y, X64Reg::RDX as _);
debug_assert_ne!(y, X64Reg::RAX as _);
debug_assert_eq!(tgt, X64Reg::RDX as _);
match (op, t) {
(Op::RemS, Type::I32) => {
dynasm! { self.ops
; cmp Rd(x), DWORD i32::MIN
; jne >do_div
; cmp Rd(y), DWORD -1
; jne >do_div
; set_zero:
; xor edx, edx
; jmp >end
; do_div:
; cdq
; idiv Rd(y)
; end:
};
}
(Op::RemU, Type::I32) => {
dynasm! { self.ops
; xor edx, edx
; div Rd(y)
};
}
(Op::RemS, Type::I64) => {
dynasm! { self.ops
; mov Rq(TMP1), QWORD i64::MIN
; cmp Rq(x), Rq(TMP1)
; jne >do_div
; mov Rq(TMP1), QWORD -1
; cmp Rq(y), Rq(TMP1)
; jne >do_div
; set_zero:
; xor rdx, rdx
; jmp >end
; do_div:
; cqo
; idiv Rq(y)
; end:
};
}
(Op::RemU, Type::I64) => {
dynasm! { self.ops
; xor rdx, rdx
; div Rq(y)
};
}
_ => unreachable!(),
}
}
fn gen_bitcnt(&mut self, op: Op, node: NodeId) {
let t = self.cfg.g[self.cfg.g[node].inputs[0]].ty;
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match (op, t) {
(Op::Clz, Type::I32) => {
dynasm! { self.ops
; cmp Rd(x), DWORD 0
; je >zero_src
; bsr Rd(tgt), Rd(x)
; not Rd(tgt)
; and Rd(tgt), DWORD 0b11111
; jmp >end
; zero_src:
; mov Rd(tgt), DWORD 32
; end:
};
}
(Op::Clz, Type::I64) => {
dynasm! { self.ops
; cmp Rq(x), DWORD 0
; je >zero_src
; bsr Rq(tgt), Rq(x)
; not Rq(tgt)
; and Rq(tgt), DWORD 0b111111
; jmp >end
; zero_src:
; mov Rq(tgt), DWORD 64
; end:
};
}
(Op::Ctz, Type::I32) => {
dynasm! { self.ops
; cmp Rd(x), DWORD 0
; je >zero_src
; bsf Rd(tgt), Rd(x)
; jmp >end
; zero_src:
; mov Rd(tgt), DWORD 32
; end:
};
}
(Op::Ctz, Type::I64) => {
dynasm! { self.ops
; cmp Rq(x), DWORD 0
; je >zero_src
; bsf Rq(tgt), Rq(x)
; jmp >end
; zero_src:
; mov Rq(tgt), DWORD 64
; end:
};
}
(Op::Popcnt, Type::I32) => dynasm!(self.ops; popcnt Rd(tgt), Rd(x)),
(Op::Popcnt, Type::I64) => dynasm!(self.ops; popcnt Rq(tgt), Rq(x)),
_ => unimplemented!(),
}
}
fn gen_cmp(&mut self, node: NodeId) {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let y = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[self.cfg.g[node].inputs[0]].ty {
Type::I32 => dynasm!(self.ops; cmp Rd(x), Rd(y)),
Type::I64 => dynasm!(self.ops; cmp Rq(x), Rq(y)),
Type::F32 => dynasm!(self.ops; comiss Rx(x), Rx(y)),
Type::F64 => dynasm!(self.ops; comisd Rx(x), Rx(y)),
t => unimplemented!("{:?}", t),
}
match (
self.cfg.g[node].op::<Op>(),
self.cfg.g[self.cfg.g[node].inputs[0]].ty,
) {
(Op::Eq, Type::I32 | Type::I64) => dynasm!(self.ops; sete Rb(tgt)),
(Op::Ne, Type::I32 | Type::I64) => dynasm!(self.ops; setne Rb(tgt)),
(Op::LtS, Type::I32 | Type::I64) => dynasm!(self.ops; setl Rb(tgt)),
(Op::LtU, Type::I32 | Type::I64) => dynasm!(self.ops; setb Rb(tgt)),
(Op::LeS, Type::I32 | Type::I64) => dynasm!(self.ops; setle Rb(tgt)),
(Op::LeU, Type::I32 | Type::I64) => dynasm!(self.ops; setbe Rb(tgt)),
(Op::GtS, Type::I32 | Type::I64) => dynasm!(self.ops; setg Rb(tgt)),
(Op::GtU, Type::I32 | Type::I64) => dynasm!(self.ops; seta Rb(tgt)),
(Op::GeS, Type::I32 | Type::I64) => dynasm!(self.ops; setge Rb(tgt)),
(Op::GeU, Type::I32 | Type::I64) => dynasm!(self.ops; setae Rb(tgt)),
(Op::Eq, Type::F32 | Type::F64) => dynasm! { self.ops
; mov Rd(TMP1), 0
; sete Rb(tgt)
; cmovp Rd(tgt), Rd(TMP1)
},
(Op::Ne, Type::F32 | Type::F64) => dynasm! { self.ops
; mov Rd(TMP1), 1
; setne Rb(tgt)
; cmovp Rd(tgt), Rd(TMP1)
},
(Op::LtF, Type::F32 | Type::F64) => dynasm! { self.ops
; mov Rd(TMP1), 0
; setb Rb(tgt)
; cmovp Rd(tgt), Rd(TMP1)
},
(Op::LeF, Type::F32 | Type::F64) => dynasm! { self.ops
; mov Rd(TMP1), 0
; setbe Rb(tgt)
; cmovp Rd(tgt), Rd(TMP1)
},
(Op::GtF, Type::F32 | Type::F64) => dynasm! { self.ops
; mov Rd(TMP1), 0
; seta Rb(tgt)
; cmovp Rd(tgt), Rd(TMP1)
},
(Op::GeF, Type::F32 | Type::F64) => dynasm! { self.ops
; mov Rd(TMP1), 0
; setae Rb(tgt)
; cmovp Rd(tgt), Rd(TMP1)
},
x => unimplemented!("{:?}", x),
}
assert_eq!(self.cfg.g[node].ty, Type::Bool);
}
fn gen_node(&mut self, node: NodeId) {
let op = self.cfg.g[node].op::<Op>();
match op {
Op::Start => self.gen_prologue(),
Op::Param if self.cfg.g[node].fixed_reg.is_none() => {
let stack_arg_index = self.cfg.g[node].stack_arg_index.unwrap() as i32;
let tgt = self.load_output_reg(node, TMP0);
let offset = 16 + (stack_arg_index << 3);
match self.cfg.g[node].ty {
Type::Bool | Type::I8 | Type::I16 | Type::I32 | Type::I64 => {
dynasm!(self.ops; mov Rq(tgt), [rbp + offset])
}
Type::F32 => dynasm!(self.ops; movss Rx(tgt), [rbp + offset]),
Type::F64 => dynasm!(self.ops; movsd Rx(tgt), [rbp + offset]),
Type::Void | Type::Top => unreachable!(),
}
}
Op::Region | Op::Param | Op::Phi | Op::EffectPhi => {
}
Op::ITruncU => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
if x != tgt {
warn!("ITruncU is not coalesced");
dynasm!(self.ops; mov Rq(tgt), Rq(x));
}
}
Op::Wrap => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
dynasm!(self.ops; mov Rd(tgt), Rd(x));
}
Op::CvtF2SI | Op::CvtF2UI | Op::CvtSI2F | Op::CvtUI2F => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match (
op,
self.cfg.g[node].ty,
self.cfg.g[self.cfg.g[node].inputs[0]].ty,
) {
(Op::CvtF2SI, Type::I32, Type::F32) => {
dynasm!(self.ops; cvttss2si Rd(tgt), Rx(x))
}
(Op::CvtF2SI, Type::I32, Type::F64) => {
dynasm!(self.ops; cvttsd2si Rd(tgt), Rx(x))
}
(Op::CvtF2SI, Type::I64, Type::F32) => {
dynasm!(self.ops; cvttss2si Rq(tgt), Rx(x))
}
(Op::CvtF2SI, Type::I64, Type::F64) => {
dynasm!(self.ops; cvttsd2si Rq(tgt), Rx(x))
}
(Op::CvtF2UI, Type::I32, Type::F32) => {
let label = self.register_constant(Const::F32(u64::MAX as _));
dynasm! { self.ops
; mov Rd(TMP1), DWORD -1
; cvtsi2ss Rx(TMP1), Rd(TMP1)
; comiss Rx(x), Rx(TMP1)
; jbe >except
; cvttss2si Rq(tgt), Rx(x)
; mov r11d, DWORD -1
; cmp Rq(tgt), r11
; ja >except
; jmp >end
; except:
; movss Rx(TMP1), [=>label]
; cvttss2si Rq(tgt), Rx(TMP1)
; end:
}
}
(Op::CvtF2UI, Type::I32, Type::F64) => {
let label = self.register_constant(Const::F32(u64::MAX as _));
dynasm! { self.ops
; mov Rd(TMP1), DWORD -1
; cvtsi2sd Rx(TMP1), Rd(TMP1)
; comisd Rx(x), Rx(TMP1)
; jbe >except
; cvttsd2si Rq(tgt), Rx(x)
; mov r11d, DWORD -1
; cmp Rq(tgt), r11
; ja >except
; jmp >end
; except:
; movss Rx(TMP1), [=>label]
; cvttss2si Rq(tgt), Rx(TMP1)
; end:
}
}
(Op::CvtF2UI, Type::I64, Type::F32) => {
let label = self.register_constant(Const::F32((i64::MAX as u64 + 1) as _));
dynasm! { self.ops
; mov Rd(TMP1), DWORD -1
; cvtsi2ss Rx(TMP1), Rd(TMP1)
; comiss Rx(x), Rx(TMP1)
; jbe >except
; movss Rx(TMP1), [=>label]
; ucomiss Rx(x), Rx(TMP1)
; jae >sub
; cvttss2si Rq(tgt), Rx(x)
; jmp >end
; sub:
; subss Rx(x), Rx(TMP1)
; cvttss2si Rq(tgt), Rx(x)
; mov Rq(TMP1), QWORD -0x8000000000000000
; or Rq(tgt), Rq(TMP1)
; jmp >end
; except:
; movss Rx(TMP1), [=>label]
; cvttss2si Rq(tgt), Rx(TMP1)
; end:
}
}
(Op::CvtF2UI, Type::I64, Type::F64) => {
let label = self.register_constant(Const::F64((i64::MAX as u64 + 1) as _));
let label2 = self.register_constant(Const::F32(u64::MAX as _));
dynasm! { self.ops
; mov Rd(TMP1), DWORD -1
; cvtsi2sd Rx(TMP1), Rd(TMP1)
; comisd Rx(x), Rx(TMP1)
; jbe >except
; movsd Rx(TMP1), [=>label]
; ucomisd Rx(x), Rx(TMP1)
; jae >sub
; cvttsd2si Rq(tgt), Rx(x)
; jmp >end
; sub:
; subsd Rx(x), Rx(TMP1)
; cvttsd2si Rq(tgt), Rx(x)
; mov Rq(TMP1), QWORD -0x8000000000000000
; or Rq(tgt), Rq(TMP1)
; jmp >end
; except:
; movss Rx(TMP1), [=>label2]
; cvttss2si Rq(tgt), Rx(TMP1)
; end:
}
}
(Op::CvtSI2F, Type::F32, Type::I32) => {
dynasm!(self.ops; pxor Rx(tgt), Rx(tgt));
dynasm!(self.ops; cvtsi2ss Rx(tgt), Rd(x));
}
(Op::CvtSI2F, Type::F32, Type::I64) => {
dynasm!(self.ops; pxor Rx(tgt), Rx(tgt));
dynasm!(self.ops; cvtsi2ss Rx(tgt), Rq(x));
}
(Op::CvtSI2F, Type::F64, Type::I32) => {
dynasm!(self.ops; pxor Rx(tgt), Rx(tgt));
dynasm!(self.ops; cvtsi2sd Rx(tgt), Rd(x));
}
(Op::CvtSI2F, Type::F64, Type::I64) => {
dynasm!(self.ops; pxor Rx(tgt), Rx(tgt));
dynasm!(self.ops; cvtsi2sd Rx(tgt), Rq(x));
}
(Op::CvtUI2F, Type::F32, Type::I32) => {
dynasm! { self.ops
; pxor Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(tgt), Rd(x)
; test Rd(x), Rd(x)
; js >s
; pxor Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(tgt), Rd(x)
; jmp >end
; s: ; push Rq(x)
; mov Rd(TMP1), Rd(x)
; and Rd(TMP1), 1
; shr Rd(x), 1
; or Rd(x), Rd(TMP1)
; pxor Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(tgt), Rd(x)
; addss Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(TMP1), Rd(TMP1)
; addss Rx(tgt), Rx(TMP1)
; pop Rq(x)
; end:
}
}
(Op::CvtUI2F, Type::F32, Type::I64) => {
dynasm! { self.ops
; pxor Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(tgt), Rq(x)
; test Rq(x), Rq(x)
; js >s
; pxor Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(tgt), Rq(x)
; jmp >end
; s: ; push Rq(x)
; mov Rd(TMP1), Rd(x)
; and Rd(TMP1), 1
; shr Rq(x), 1
; or Rq(x), Rq(TMP1)
; pxor Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(tgt), Rq(x)
; addss Rx(tgt), Rx(tgt)
; cvtsi2ss Rx(TMP1), Rd(TMP1)
; addss Rx(tgt), Rx(TMP1)
; pop Rq(x)
; end:
}
}
(Op::CvtUI2F, Type::F64, Type::I32) => {
dynasm! { self.ops
; pxor Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(tgt), Rd(x)
; test Rd(x), Rd(x)
; js >s
; pxor Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(tgt), Rd(x)
; jmp >end
; s: ; push Rq(x)
; mov Rd(TMP1), Rd(x)
; and Rd(TMP1), 1
; shr Rd(x), 1
; or Rd(x), Rd(TMP1)
; pxor Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(tgt), Rd(x)
; addsd Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(TMP1), Rd(TMP1)
; addsd Rx(tgt), Rx(TMP1)
; pop Rq(x)
; end:
}
}
(Op::CvtUI2F, Type::F64, Type::I64) => {
dynasm! { self.ops
; pxor Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(tgt), Rq(x)
; test Rq(x), Rq(x)
; js >s
; pxor Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(tgt), Rq(x)
; jmp >end
; s: ; push Rq(x)
; mov Rd(TMP1), Rd(x)
; and Rd(TMP1), 1
; shr Rq(x), 1
; or Rq(x), Rq(TMP1)
; pxor Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(tgt), Rq(x)
; addsd Rx(tgt), Rx(tgt)
; cvtsi2sd Rx(TMP1), Rd(TMP1)
; addsd Rx(tgt), Rx(TMP1)
; pop Rq(x)
; end:
}
}
_ => unreachable!(),
}
}
Op::CvtF2F => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
if self.cfg.g[self.cfg.g[node].inputs[0]].ty == self.cfg.g[node].ty {
} else if self.cfg.g[node].ty == Type::F64 {
dynasm!(self.ops; cvtss2sd Rx(tgt), Rx(x));
} else {
dynasm!(self.ops; cvtsd2ss Rx(tgt), Rx(x));
}
}
Op::Bitcast => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match (
self.cfg.g[node].ty,
self.cfg.g[self.cfg.g[node].inputs[0]].ty,
) {
(Type::F32, Type::I32) => dynasm!(self.ops; movd Rx(tgt), Rd(x)),
(Type::I32, Type::F32) => dynasm!(self.ops; movd Rd(tgt), Rx(x)),
(Type::F64, Type::I64) => dynasm!(self.ops; movq Rx(tgt), Rq(x)),
(Type::I64, Type::F64) => dynasm!(self.ops; movq Rq(tgt), Rx(x)),
_ => unreachable!(),
}
}
Op::Const => {
let v = match self.cfg.g[node].literal.as_ref().unwrap() {
Literal::Value(v) => v.clone(),
_ => unreachable!(),
};
let tgt = self.load_output_reg(node, TMP0);
match v {
Const::Bool(v) => dynasm!(self.ops; mov Rb(tgt), BYTE v as _),
Const::I32(v) => dynasm!(self.ops; mov Rd(tgt), DWORD v as _),
Const::I64(v) => dynasm!(self.ops; fs mov Rq(tgt), QWORD v as _),
Const::F32(v) => {
let v = unsafe { std::mem::transmute::<f32, i32>(v) };
dynasm! {self.ops
; mov Rd(TMP1), DWORD v
; movd Rx(tgt), Rd(TMP1)
}
}
Const::F64(v) => {
let v = unsafe { std::mem::transmute::<f64, i64>(v) };
dynasm! {self.ops
; mov Rq(TMP1), QWORD v
; movq Rx(tgt), Rq(TMP1)
}
}
_ => unimplemented!("{:?}", v),
}
}
Op::Move => {
let o = &self.cfg.liveness[self.cfg.g[node].interval];
let i = &self.cfg.liveness[self.cfg.g[self.cfg.g[node].inputs[0]].interval];
if !((i.reg.is_some() && i.reg == o.reg)
|| (i.reg.is_none() && o.reg.is_none() && i.mem == o.mem))
{
if o.reg.is_some() && i.reg.is_some() {
self.gen_mov_reg_reg(
self.cfg.g[node].ty,
o.reg.unwrap() as _,
i.reg.unwrap() as _,
);
} else if o.reg.is_none() && i.reg.is_none() {
let (i_mem, o_mem) = (i.mem, o.mem);
self.gen_mov_reg_stack(self.cfg.g[node].ty, TMP0, i_mem);
self.gen_mov_stack_reg(self.cfg.g[node].ty, TMP0, o_mem);
} else if o.reg.is_none() {
self.gen_mov_stack_reg(self.cfg.g[node].ty, i.reg.unwrap() as _, o.mem);
} else if !o.reg.is_none() && i.reg.is_none() {
self.gen_mov_reg_stack(self.cfg.g[node].ty, o.reg.unwrap() as _, i.mem);
}
}
}
Op::Not => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
assert_eq!(self.cfg.g[node].ty, Type::Bool);
dynasm! { self.ops
; mov Rb(tgt), Rb(x)
; xor Rb(tgt), 1
};
}
Op::Neg => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[node].ty {
Type::I8 | Type::I16 | Type::I32 => {
if x != tgt {
dynasm!(self.ops; mov Rd(tgt), Rd(x));
}
dynasm!(self.ops; neg Rd(tgt));
}
Type::I64 => {
if x != tgt {
dynasm!(self.ops; mov Rq(tgt), Rq(x));
}
dynasm!(self.ops; neg Rq(tgt));
}
Type::F32 => {
if x != tgt {
dynasm!(self.ops; movss Rx(tgt), Rx(x));
}
let label = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x8000000080000000_8000000080000000u128)
}));
dynasm! {self.ops
; xorps Rx(tgt), [=>label]
}
}
Type::F64 => {
if x != tgt {
dynasm!(self.ops; movsd Rx(tgt), Rx(x));
}
let label = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x8000000000000000_8000000000000000u128)
}));
dynasm! {self.ops
; xorpd Rx(tgt), [=>label]
}
}
_ => unreachable!(),
}
}
Op::Add | Op::Sub | Op::Mul | Op::And | Op::Or | Op::Xor => self.gen_binop(op, node),
Op::FDiv => self.gen_binop(op, node),
Op::DivS | Op::DivU => self.gen_div(op, node),
Op::RemS | Op::RemU => self.gen_irem(op, node),
Op::Shl | Op::ShrS | Op::ShrU | Op::Rotl | Op::Rotr => {
self.gen_shift_op(op, node, X64Reg::RCX)
}
Op::Clz | Op::Ctz | Op::Popcnt => self.gen_bitcnt(op, node),
Op::SExt => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match (
self.cfg.g[self.cfg.g[node].inputs[0]].ty,
self.cfg.g[node].ty,
) {
(Type::I8 | Type::Bool, Type::I32) => dynasm!(self.ops; movsx Rd(tgt), Rb(x)),
(Type::I16, Type::I32) => dynasm!(self.ops; movsx Rd(tgt), Rw(x)),
(Type::I8 | Type::Bool, Type::I64) => dynasm!(self.ops; movsx Rq(tgt), Rb(x)),
(Type::I16, Type::I64) => dynasm!(self.ops; movsx Rq(tgt), Rw(x)),
(Type::I32, Type::I64) => dynasm!(self.ops; movsx Rq(tgt), Rd(x)),
_ => unimplemented!(),
}
}
Op::ZExt => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match (
self.cfg.g[self.cfg.g[node].inputs[0]].ty,
self.cfg.g[node].ty,
) {
(Type::I8 | Type::Bool, Type::I32) => dynasm!(self.ops; movzx Rd(tgt), Rb(x)),
(Type::I16, Type::I32) => dynasm!(self.ops; movzx Rd(tgt), Rw(x)),
(Type::I8 | Type::Bool, Type::I64) => dynasm!(self.ops; movzx Rq(tgt), Rb(x)),
(Type::I16, Type::I64) => dynasm!(self.ops; movzx Rq(tgt), Rw(x)),
(Type::I32, Type::I64) => dynasm!(self.ops; mov Rd(tgt), Rd(x)),
_ => unimplemented!(),
}
}
Op::FSqrt => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[node].ty {
Type::F32 => dynasm!(self.ops; sqrtps Rx(tgt), Rx(x)),
Type::F64 => dynasm!(self.ops; sqrtpd Rx(tgt), Rx(x)),
_ => unimplemented!(),
}
}
Op::FRound | Op::FFloor | Op::FCeil | Op::FTrunc => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match (op, self.cfg.g[node].ty) {
(Op::FRound, Type::F32) => dynasm!(self.ops; roundss Rx(tgt), Rx(x), BYTE 0b00),
(Op::FRound, Type::F64) => dynasm!(self.ops; roundsd Rx(tgt), Rx(x), BYTE 0b00),
(Op::FFloor, Type::F32) => dynasm!(self.ops; roundss Rx(tgt), Rx(x), BYTE 0b01),
(Op::FFloor, Type::F64) => dynasm!(self.ops; roundsd Rx(tgt), Rx(x), BYTE 0b01),
(Op::FCeil, Type::F32) => dynasm!(self.ops; roundss Rx(tgt), Rx(x), BYTE 0b10),
(Op::FCeil, Type::F64) => dynasm!(self.ops; roundsd Rx(tgt), Rx(x), BYTE 0b10),
(Op::FTrunc, Type::F32) => dynasm!(self.ops; roundss Rx(tgt), Rx(x), BYTE 0b11),
(Op::FTrunc, Type::F64) => dynasm!(self.ops; roundsd Rx(tgt), Rx(x), BYTE 0b11),
_ => unimplemented!(),
}
}
Op::FAbs => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[node].ty {
Type::F32 => {
let label = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x8000000080000000_8000000080000000u128)
}));
if x != tgt {
dynasm!(self.ops; movss Rx(tgt), Rx(x));
}
dynasm! {self.ops
; pxor Rx(TMP1), Rx(TMP1)
; comiss Rx(tgt), Rx(TMP1)
; jb >neg
; jmp >end
; neg:
; xorps Rx(tgt), [=>label]
; end:
}
}
Type::F64 => {
let label = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x8000000000000000_8000000000000000u128)
}));
if x != tgt {
dynasm!(self.ops; movsd Rx(tgt), Rx(x));
}
dynasm! {self.ops
; pxor Rx(TMP1), Rx(TMP1)
; comisd Rx(tgt), Rx(TMP1)
; jb >neg
; jmp >end
; neg:
; xorpd Rx(tgt), [=>label]
; end:
}
}
_ => unimplemented!(),
}
}
Op::FCopysign => {
let mag = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let sgn = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[node].ty {
Type::F32 => {
let label = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x8000000080000000_8000000080000000u128)
}));
let label2 = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x7fffffff7fffffff_7fffffff7fffffffu128)
}));
if mag != tgt {
dynasm!(self.ops; movss Rx(tgt), Rx(mag));
}
dynasm! {self.ops
; andps Rx(sgn), [=>label]
; andps Rx(tgt), [=>label2]
; orps Rx(tgt), Rx(sgn)
}
}
Type::F64 => {
let label = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x8000000000000000_8000000000000000u128)
}));
let label2 = self.register_constant(Const::I128(unsafe {
std::mem::transmute(0x7fffffffffffffff_7fffffffffffffffu128)
}));
if mag != tgt {
dynasm!(self.ops; movsd Rx(tgt), Rx(mag));
}
dynasm! {self.ops
; andpd Rx(sgn), [=>label]
; andpd Rx(tgt), [=>label2]
; orpd Rx(tgt), Rx(sgn)
}
}
_ => unimplemented!(),
}
}
Op::Load => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[node].ty {
Type::I8 => dynasm!(self.ops; mov Rb(tgt), [Rq(x)]),
Type::I16 => dynasm!(self.ops; mov Rw(tgt), [Rq(x)]),
Type::I32 => dynasm!(self.ops; mov Rd(tgt), [Rq(x)]),
Type::I64 => dynasm!(self.ops; mov Rq(tgt), [Rq(x)]),
Type::F32 => dynasm!(self.ops; movss Rx(tgt), [Rq(x)]),
Type::F64 => dynasm!(self.ops; movsd Rx(tgt), [Rq(x)]),
_ => unimplemented!(),
}
}
Op::Store => {
let ptr = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let val = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
match self.cfg.g[self.cfg.g[node].inputs[1]].ty {
Type::I8 => dynasm!(self.ops; mov [Rq(ptr)], Rb(val)),
Type::I16 => dynasm!(self.ops; mov [Rq(ptr)], Rw(val)),
Type::I32 => dynasm!(self.ops; mov [Rq(ptr)], Rd(val)),
Type::I64 => dynasm!(self.ops; mov [Rq(ptr)], Rq(val)),
Type::F32 => dynasm!(self.ops; movss [Rq(ptr)], Rx(val)),
Type::F64 => dynasm!(self.ops; movsd [Rq(ptr)], Rx(val)),
_ => unimplemented!("{:?}", self.cfg.g[node].ty),
}
}
Op::Eq
| Op::Ne
| Op::LtS
| Op::LtU
| Op::LeS
| Op::LeU
| Op::GtS
| Op::GtU
| Op::GeS
| Op::GeU
| Op::LtF
| Op::LeF
| Op::GtF
| Op::GeF => self.gen_cmp(node),
Op::IsNan => {
let x = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let tgt = self.load_output_reg(node, TMP0);
match self.cfg.g[self.cfg.g[node].inputs[0]].ty {
Type::F32 => dynasm!(self.ops; ucomiss Rx(x), Rx(x)),
Type::F64 => dynasm!(self.ops; ucomisd Rx(x), Rx(x)),
t => unimplemented!("{:?}", t),
}
dynasm!(self.ops; setp Rb(tgt));
assert_eq!(self.cfg.g[node].ty, Type::Bool);
}
Op::Call | Op::CallIndirect => {
let mut saved_intervals = vec![];
for i in 0..self.cfg.liveness.len() {
let interval = &self.cfg.liveness[i];
if i != self.cfg.g[node].interval
&& interval.covers(self.cfg.g[node].cfg_id)
&& interval.reg.is_some()
{
saved_intervals.push(i);
self.save_interval(i);
}
}
let args_start_index = match op {
Op::CallIndirect if self.cfg.g[node].has_call_indirect_ctx => 2,
Op::CallIndirect if !self.cfg.g[node].has_call_indirect_ctx => 1,
_ => 0,
};
let fptr = if op == Op::CallIndirect {
let fptr = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
if self.cfg.g[node].has_call_indirect_ctx {
let _ctx = self.load_input_reg(self.cfg.g[node].inputs[1], TMP1);
}
Some(fptr)
} else {
None
};
let mut stack_args = vec![];
for i in 0..self.cfg.g[node].inputs[args_start_index..].len() {
let a = self.cfg.g[node].inputs[args_start_index + i];
if let Some(fixed_reg) = self.cfg.g[a].fixed_reg {
let fixed_reg = fixed_reg as u8;
if self.cfg.g[a].fixed_reg != self.cfg.liveness[self.cfg.g[a].interval].reg
{
if let Some(reg) = self.cfg.liveness[self.cfg.g[a].interval].reg {
self.gen_mov_reg_reg(self.cfg.g[a].ty, fixed_reg, reg as u8);
} else {
let mem = self.cfg.liveness[self.cfg.g[a].interval].mem;
self.gen_mov_reg_stack(self.cfg.g[a].ty, fixed_reg, mem);
}
}
} else {
stack_args.push(a);
}
}
if !stack_args.is_empty() {
let rsp_delta = stack_args.len() << 3;
for (i, a) in stack_args.iter().enumerate() {
self.gen_push(*a, i as _, rsp_delta as i32);
}
dynasm!(self.ops; sub rsp, DWORD rsp_delta as i32);
}
if op == Op::CallIndirect {
dynasm!(self.ops; call Rq(fptr.unwrap()));
} else if let Literal::Func(Symbol::ExternFn(ptr)) =
self.cfg.g[node].literal.clone().unwrap()
{
dynasm! { self.ops
; fs mov rax, QWORD ptr as i64
; call rax
};
} else {
dynasm!(self.ops; call DWORD 0);
let symbol = match self.cfg.g[node].literal.as_ref().unwrap() {
Literal::Func(name) => name.clone(),
_ => unreachable!(),
};
let offset = self.ops.offset().0;
self.record_static_callsite(symbol, offset);
}
debug_assert!(self.cfg.g[node].fixed_reg.is_some());
if self.cfg.g[node].fixed_reg != self.cfg.liveness[self.cfg.g[node].interval].reg {
let fixed_reg = self.cfg.g[node].fixed_reg.unwrap() as u8;
if let Some(reg) = self.cfg.liveness[self.cfg.g[node].interval].reg {
self.gen_mov_reg_reg(self.cfg.g[node].ty, fixed_reg, reg as u8);
} else {
let mem = self.cfg.liveness[self.cfg.g[node].interval].mem;
self.gen_mov_stack_reg(self.cfg.g[node].ty, fixed_reg, mem);
}
}
for i in saved_intervals {
self.restore_interval(i);
}
let stack_args_count = stack_args.len();
if stack_args_count > 0 {
dynasm!(self.ops; add rsp, DWORD (stack_args_count << 3) as i32);
}
}
Op::Branch => {
let cond = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let true_target = self.get_successor_label(0);
let false_target = self.get_successor_label(1);
let next_block = self.current_block_id + 1;
let succ = &self.cfg.blocks[self.current_block_id].succs;
if succ[0] == next_block {
dynasm! { self.ops
; cmp Rb(cond), BYTE 0
; je => false_target
};
} else if succ[1] == next_block {
dynasm! { self.ops
; cmp Rb(cond), BYTE 0
; jne => true_target
};
} else {
unreachable!()
}
}
Op::Jump => {
let target = self.get_successor_label(0);
if self.get_successor_id(0) == self.current_block_id + 1 {
} else {
dynasm! { self.ops
; jmp => target
};
}
}
Op::BrTable => {
let index = self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
let default_use_index = self.cfg.g[node].control_uses.len() - 1;
let default_target = self.get_successor_label(default_use_index);
if default_use_index > 0 {
dynasm! { self.ops
; cmp Rd(index), DWORD default_use_index as i32
; jae => default_target
};
dynasm! { self.ops
; lea Rq(TMP0), [Rd(index) + Rd(index) * 4]
; lea Rq(TMP1), [>jump_table]
; add Rq(TMP1), Rq(TMP0)
; jmp Rq(TMP1)
; jump_table:
};
for i in 0..default_use_index {
let target = self.get_successor_label(i);
dynasm!(self.ops; jmp => target);
}
} else {
dynasm!(self.ops; jmp => default_target);
}
}
Op::Return => {
if self.cfg.g[node].inputs.len() > 0 {
debug_assert_eq!(self.cfg.g[node].inputs.len(), 1);
self.load_input_reg(self.cfg.g[node].inputs[0], TMP0);
}
self.gen_epilogue()
}
Op::DebugBreak => dynasm!(self.ops; int3),
_ => unimplemented!("{:?}", node),
}
if op != Op::Call
&& op != Op::CallIndirect
&& op != Op::Move
&& op != Op::Phi
&& op != Op::EffectPhi
{
self.update_spilled_values(node)
}
}
pub fn gen(mut self) -> Arc<dyn CompiledCode> {
for _ in 0..self.cfg.blocks.len() {
self.labels.push(self.ops.new_dynamic_label());
}
for i in 0..self.cfg.blocks.len() {
self.current_block_id = self.cfg.blocks[i].id;
let label = self.labels[self.cfg.blocks[i].id];
dynasm! { self.ops
; => label
};
self.gen_node(self.cfg.blocks[i].label.cast());
for j in 0..self.cfg.blocks[i].nodes.len() {
self.gen_node(self.cfg.blocks[i].nodes[j]);
}
self.gen_node(self.cfg.blocks[i].terminal.cast());
}
for (constant, label) in std::mem::take(&mut self.constants) {
match constant {
Const::I128(v) => {
dynasm! {self.ops
; .align 16
; => label
; .bytes v.to_le_bytes()
};
}
Const::F32(v) => {
dynasm! {self.ops
; .align 8
; => label
; .bytes v.to_le_bytes()
};
}
Const::F64(v) => {
dynasm! {self.ops
; .align 16
; => label
; .bytes v.to_le_bytes()
};
}
_ => unreachable!(),
}
}
let buf = self.ops.finalize().unwrap();
let buf = self.compiler.code.alloc_code_buffer::<X64>(&buf);
let mut call_sites = HashMap::<usize, Box<dyn CallSite>>::new();
for (offset, mut call_site) in std::mem::take(&mut self.static_call_sites) {
call_site.set_start_address(buf.start());
call_sites.insert(offset, call_site);
}
let code = X64CompiledCode::new(self.symbol, self.cfg.signature.clone(), buf, call_sites);
code.link_lazy_compilation_trampoline(self.compiler.lazy_compilation_trampoline.start());
code.dump();
self.compiler.code.register_code_object(code)
}
}